From e9835a7827a4941bf12150324e7a544be6f89fdf Mon Sep 17 00:00:00 2001 From: William Bernbaum Date: Tue, 25 Aug 2026 05:17:02 -0700 Subject: [PATCH v1] Skipping NULL keys when uniqueifying a semijoin's RHS When the planner unique-ifies the righthand side of a semijoin, it reads every righthand row. A row whose join key is NULL can never match, since the operators unique-ification uses are btree or hash equality and those are strict. Such a row is sorted or hashed, deduplicated, and only then discarded at the join. Add add_semijoin_not_null_quals(), which walks root->join_info_list and, for each semijoin whose righthand keys can be unique-ified, pushes an IS NOT NULL baserestriction onto the key. It runs from query_planner() before set_base_rel_sizes(), so the row counts and distinct-value estimates the join search works from already account for the rows that will be thrown away. That is frequently enough for unique-ification to stop looking worthwhile, in which case the planner returns to a plain semi join. The pass declines to act in several cases. The key has to resolve to a single base relation, a baserestriction being what keeps rows out of a scan; pull_varnos() counts outer-join relids, so a key that an outer join can null drops out here as well. Subqueries and other non-RTE_RELATION entries are skipped, their output nullability not yet being known at this stage. Columns the catalog already declares NOT NULL need no test. clause_selectivity() is consulted first, so a column that is nullable on paper but whose statistics show no NULLs does not pay for a test that rejects nothing. And predicate_implied_by() suppresses a test that an existing strict qual already implies, which keeps its selectivity from being counted twice and keeps two semijoins over the same expression from each contributing a copy. --- src/backend/optimizer/plan/initsplan.c | 100 ++++++++++++++++++ src/backend/optimizer/plan/planmain.c | 7 ++ src/include/optimizer/planmain.h | 1 + .../regress/expected/collate.icu.utf8.out | 7 +- src/test/regress/expected/join.out | 39 +++---- src/test/regress/expected/partition_join.out | 25 +++-- src/test/regress/expected/subselect.out | 14 +-- 7 files changed, 155 insertions(+), 38 deletions(-) diff --git a/src/backend/optimizer/plan/initsplan.c b/src/backend/optimizer/plan/initsplan.c index f08a918146c..6318d71a584 100644 --- a/src/backend/optimizer/plan/initsplan.c +++ b/src/backend/optimizer/plan/initsplan.c @@ -2651,6 +2651,106 @@ compute_semijoin_info(PlannerInfo *root, SpecialJoinInfo *sjinfo, List *clause) sjinfo->semi_rhs_exprs = semi_rhs_exprs; } +/* + * add_semijoin_not_null_quals + * Reject righthand rows whose semijoin key is NULL + * + * A semijoin's righthand side is often unique-ified before the join runs, and + * unique-ification reads every righthand row. A row whose key is NULL never + * matches. We should reject such rows upfront. + * + * set_base_rel_sizes() must see these quals, so this runs before the join + * search begins. + */ +void +add_semijoin_not_null_quals(PlannerInfo *root) +{ + ListCell *lc; + + foreach(lc, root->join_info_list) + { + SpecialJoinInfo *sjinfo = (SpecialJoinInfo *) lfirst(lc); + ListCell *lc2; + + if (sjinfo->jointype != JOIN_SEMI) + continue; + + /* Nothing reads the whole RHS unless it can be unique-ified */ + if (!sjinfo->semi_can_btree && !sjinfo->semi_can_hash) + continue; + + foreach(lc2, sjinfo->semi_rhs_exprs) + { + Expr *expr = (Expr *) lfirst(lc2); + Relids relids; + RelOptInfo *rel; + NullTest *ntest; + RestrictInfo *rinfo; + + /* + * A baserestriction keeps rows out of the scan, so the expression + * must belong to one relation. pull_varnos() counts outer-join + * relids too, so a key nulled by an outer join drops out here as + * well. + */ + relids = pull_varnos(root, (Node *) expr); + if (bms_membership(relids) != BMS_SINGLETON) + continue; + rel = root->simple_rel_array[bms_singleton_member(relids)]; + if (rel == NULL) + continue; + + /* + * A subquery has not been planned yet, so the key's nullability is + * unknown. + */ + if (rel->rtekind != RTE_RELATION) + continue; + + + /* The catalog already forbids nulls */ + if (IsA(expr, Var)) + { + Var *var = (Var *) expr; + + if (var->varattno > 0 && + bms_is_member(var->varattno, rel->notnullattnums)) + continue; + } + + ntest = makeNode(NullTest); + ntest->arg = copyObject(expr); + ntest->nulltesttype = IS_NOT_NULL; + ntest->argisrow = false; /* correct even if composite arg */ + ntest->location = -1; + + /* + * Statistics say there is nothing to reject. This also covers a + * righthand side the catalog check above cannot see into, such as + * a subquery that survived pullup. + */ + if (clause_selectivity(root, (Node *) ntest, 0, + JOIN_INNER, NULL) >= 1.0) + continue; + + /* + * A strict qual on the same expression rejects these rows already, + * and adding a second test would count its selectivity twice. + */ + if (predicate_implied_by(list_make1(ntest), + extract_actual_clauses(rel->baserestrictinfo, + false), + false)) + continue; + + rinfo = make_restrictinfo(root, (Expr *) ntest, true, false, false, + false, root->qual_security_level, + relids, NULL, NULL); + distribute_restrictinfo_to_rels(root, rinfo); + } + } +} + /* * deconstruct_distribute_oj_quals * Adjust LEFT JOIN quals to be suitable for commuted-left-join cases, diff --git a/src/backend/optimizer/plan/planmain.c b/src/backend/optimizer/plan/planmain.c index 02495e22e24..632f8d82c27 100644 --- a/src/backend/optimizer/plan/planmain.c +++ b/src/backend/optimizer/plan/planmain.c @@ -241,6 +241,13 @@ query_planner(PlannerInfo *root, */ joinlist = remove_useless_self_joins(root, joinlist); + /* + * Every semijoin now known can reject righthand rows whose join key is + * NULL. This has to precede appendrel expansion so that a child relation + * inherits the restriction. + */ + add_semijoin_not_null_quals(root); + /* * Now distribute "placeholders" to base rels as needed. This has to be * done after join removal because removal could change whether a diff --git a/src/include/optimizer/planmain.h b/src/include/optimizer/planmain.h index 71c043a25e8..f245bb5f704 100644 --- a/src/include/optimizer/planmain.h +++ b/src/include/optimizer/planmain.h @@ -87,6 +87,7 @@ extern bool restriction_is_always_false(PlannerInfo *root, RestrictInfo *restrictinfo); extern void distribute_restrictinfo_to_rels(PlannerInfo *root, RestrictInfo *restrictinfo); +extern void add_semijoin_not_null_quals(PlannerInfo *root); extern RestrictInfo *process_implied_equality(PlannerInfo *root, Oid opno, Oid collation, diff --git a/src/test/regress/expected/collate.icu.utf8.out b/src/test/regress/expected/collate.icu.utf8.out index fcfcc658bea..ac9166e727b 100644 --- a/src/test/regress/expected/collate.icu.utf8.out +++ b/src/test/regress/expected/collate.icu.utf8.out @@ -1785,8 +1785,8 @@ EXPLAIN (COSTS OFF) SELECT * FROM test3cs t1 WHERE EXISTS (SELECT 1 FROM test3cs t2 WHERE t1.x = t2.x COLLATE case_insensitive) ORDER BY 1; - QUERY PLAN --------------------------------------------------- + QUERY PLAN +----------------------------------------------------- Sort Sort Key: t1.x COLLATE case_sensitive -> Hash Semi Join @@ -1794,7 +1794,8 @@ ORDER BY 1; -> Seq Scan on test3cs t1 -> Hash -> Seq Scan on test3cs t2 -(7 rows) + Filter: ((x)::text IS NOT NULL) +(8 rows) SELECT * FROM test3cs t1 WHERE EXISTS (SELECT 1 FROM test3cs t2 WHERE t1.x = t2.x COLLATE case_insensitive) diff --git a/src/test/regress/expected/join.out b/src/test/regress/expected/join.out index 9261e13442d..725b5f1843e 100644 --- a/src/test/regress/expected/join.out +++ b/src/test/regress/expected/join.out @@ -3053,17 +3053,18 @@ select * from tbl_rs t1 join lateral (select * from tbl_rs t2 where t2.a in (select t1.a+t3.a from tbl_rs t3) and t2.a < 5) on true; - QUERY PLAN -------------------------------------------- + QUERY PLAN +------------------------------------------------ Nested Loop -> Seq Scan on tbl_rs t1 -> Hash Right Semi Join Hash Cond: ((t1.a + t3.a) = t2.a) -> Seq Scan on tbl_rs t3 + Filter: ((t1.a + a) IS NOT NULL) -> Hash -> Seq Scan on tbl_rs t2 Filter: (a < 5) -(8 rows) +(9 rows) -- and check we get the expected results select * from tbl_rs t1 join @@ -3518,8 +3519,8 @@ select * from tenk1 t1 left join (select t2.c from tbl_anti t2 where exists (select 1 from tbl_anti t3 where t2.c = t3.c)) ss on true where ss.c is null; - QUERY PLAN -------------------------------------------------------- + QUERY PLAN +--------------------------------------------------------- Nested Loop Anti Join -> Seq Scan on tenk1 t1 -> Materialize @@ -3530,7 +3531,8 @@ where ss.c is null; -> HashAggregate Group Key: t3.c -> Seq Scan on tbl_anti t3 -(10 rows) + Filter: (c IS NOT NULL) +(11 rows) -- this is not an antijoin: the join clause t2.c = t3.c cannot guarantee t3.c -- is non-null @@ -7710,17 +7712,18 @@ where exists (select 1 from t t4 Group Key: t5.a -> Hash Join Output: t5.a - Hash Cond: (t6.b = t4.b) - -> Seq Scan on pg_temp.t t6 - Output: t6.a, t6.b + Hash Cond: (t5.b = t4.b) + -> Seq Scan on pg_temp.t t5 + Output: t5.a, t5.b + Filter: (t5.a IS NOT NULL) -> Hash - Output: t4.b, t5.b, t5.a + Output: t4.b, t6.b -> Hash Join - Output: t4.b, t5.b, t5.a + Output: t4.b, t6.b Inner Unique: true - Hash Cond: (t5.b = t4.b) - -> Seq Scan on pg_temp.t t5 - Output: t5.a, t5.b + Hash Cond: (t6.b = t4.b) + -> Seq Scan on pg_temp.t t6 + Output: t6.a, t6.b -> Hash Output: t4.b, t4.a -> Index Scan using t_a_key on pg_temp.t t4 @@ -7729,7 +7732,7 @@ where exists (select 1 from t t4 -> Index Only Scan using t_a_key on pg_temp.t t3 Output: t3.a Index Cond: (t3.a = t5.a) -(32 rows) +(33 rows) select t1.a from t t1 left join t t2 on t1.a = t2.a @@ -8223,8 +8226,8 @@ select t4.a from sj t1 explain (verbose, costs off) select t1.a from sj t1 where t1.b in ( select t2.b from sj t2 join sj t3 on t2.c=t3.c); - QUERY PLAN ------------------------------------------- + QUERY PLAN +------------------------------------------------------------------- Nested Loop Semi Join Output: t1.a Join Filter: (t1.b = t3.b) @@ -8234,7 +8237,7 @@ select t1.a from sj t1 where t1.b in ( Output: t3.c, t3.b -> Seq Scan on public.sj t3 Output: t3.c, t3.b - Filter: (t3.c IS NOT NULL) + Filter: ((t3.c IS NOT NULL) AND (t3.b IS NOT NULL)) (10 rows) -- diff --git a/src/test/regress/expected/partition_join.out b/src/test/regress/expected/partition_join.out index 38643d41fd7..a167fae3b63 100644 --- a/src/test/regress/expected/partition_join.out +++ b/src/test/regress/expected/partition_join.out @@ -1194,8 +1194,8 @@ SELECT t1.* FROM prt1 t1 WHERE t1.a IN (SELECT t1.b FROM prt2 t1, prt1_e t2 WHER EXPLAIN (COSTS OFF) SELECT t1.* FROM prt1 t1 WHERE t1.a IN (SELECT t1.b FROM prt2 t1 WHERE t1.b IN (SELECT (t1.a + t1.b)/2 FROM prt1_e t1 WHERE t1.c = 0)) AND t1.b = 0 ORDER BY t1.a; - QUERY PLAN ---------------------------------------------------------------------------- + QUERY PLAN +----------------------------------------------------------------------------------------- Merge Append Sort Key: t1.a -> Nested Loop @@ -1207,7 +1207,7 @@ SELECT t1.* FROM prt1 t1 WHERE t1.a IN (SELECT t1.b FROM prt2 t1 WHERE t1.b IN ( -> Seq Scan on prt2_p1 t1_6 -> Hash -> Seq Scan on prt1_e_p1 t1_9 - Filter: (c = 0) + Filter: ((c = 0) AND (((a + b) / 2) IS NOT NULL)) -> Index Scan using iprt1_p1_a on prt1_p1 t1_3 Index Cond: (a = t1_6.b) Filter: (b = 0) @@ -1220,7 +1220,7 @@ SELECT t1.* FROM prt1 t1 WHERE t1.a IN (SELECT t1.b FROM prt2 t1 WHERE t1.b IN ( -> Seq Scan on prt2_p2 t1_7 -> Hash -> Seq Scan on prt1_e_p2 t1_10 - Filter: (c = 0) + Filter: ((c = 0) AND (((a + b) / 2) IS NOT NULL)) -> Index Scan using iprt1_p2_a on prt1_p2 t1_4 Index Cond: (a = t1_7.b) Filter: (b = 0) @@ -1233,7 +1233,7 @@ SELECT t1.* FROM prt1 t1 WHERE t1.a IN (SELECT t1.b FROM prt2 t1 WHERE t1.b IN ( -> Seq Scan on prt2_p3 t1_8 -> Hash -> Seq Scan on prt1_e_p3 t1_11 - Filter: (c = 0) + Filter: ((c = 0) AND (((a + b) / 2) IS NOT NULL)) -> Index Scan using iprt1_p3_a on prt1_p3 t1_5 Index Cond: (a = t1_8.b) Filter: (b = 0) @@ -1254,8 +1254,8 @@ SET enable_hashjoin TO off; SET enable_nestloop TO off; EXPLAIN (COSTS OFF) SELECT t1.* FROM prt1 t1 WHERE t1.a IN (SELECT t1.b FROM prt2 t1 WHERE t1.b IN (SELECT (t1.a + t1.b)/2 FROM prt1_e t1 WHERE t1.c = 0)) ORDER BY t1.a; - QUERY PLAN ------------------------------------------------------------------- + QUERY PLAN +----------------------------------------------------------------------------- Merge Append Sort Key: t1.a -> Merge Semi Join @@ -1271,7 +1271,7 @@ SELECT t1.* FROM prt1 t1 WHERE t1.a IN (SELECT t1.b FROM prt2 t1 WHERE t1.b IN ( -> Sort Sort Key: (((t1_9.a + t1_9.b) / 2)) -> Seq Scan on prt1_e_p1 t1_9 - Filter: (c = 0) + Filter: ((c = 0) AND (((a + b) / 2) IS NOT NULL)) -> Merge Semi Join Merge Cond: (t1_4.a = t1_7.b) -> Sort @@ -1285,7 +1285,7 @@ SELECT t1.* FROM prt1 t1 WHERE t1.a IN (SELECT t1.b FROM prt2 t1 WHERE t1.b IN ( -> Sort Sort Key: (((t1_10.a + t1_10.b) / 2)) -> Seq Scan on prt1_e_p2 t1_10 - Filter: (c = 0) + Filter: ((c = 0) AND (((a + b) / 2) IS NOT NULL)) -> Merge Semi Join Merge Cond: (t1_5.a = t1_8.b) -> Sort @@ -1299,7 +1299,7 @@ SELECT t1.* FROM prt1 t1 WHERE t1.a IN (SELECT t1.b FROM prt2 t1 WHERE t1.b IN ( -> Sort Sort Key: (((t1_11.a + t1_11.b) / 2)) -> Seq Scan on prt1_e_p3 t1_11 - Filter: (c = 0) + Filter: ((c = 0) AND (((a + b) / 2) IS NOT NULL)) (44 rows) SELECT t1.* FROM prt1 t1 WHERE t1.a IN (SELECT t1.b FROM prt2 t1 WHERE t1.b IN (SELECT (t1.a + t1.b)/2 FROM prt1_e t1 WHERE t1.c = 0)) ORDER BY t1.a; @@ -4135,22 +4135,25 @@ SELECT t1.* FROM plt1_adv t1 WHERE EXISTS (SELECT 1 FROM plt2_adv t2 WHERE t1.a -> Hash Right Semi Join Hash Cond: ((t2_1.a = t1_1.a) AND (t2_1.c = t1_1.c)) -> Seq Scan on plt2_adv_p1 t2_1 + Filter: (c IS NOT NULL) -> Hash -> Seq Scan on plt1_adv_p1_null t1_1 Filter: (b < 10) -> Hash Right Semi Join Hash Cond: ((t2_2.a = t1_2.a) AND (t2_2.c = t1_2.c)) -> Seq Scan on plt2_adv_p2 t2_2 + Filter: (c IS NOT NULL) -> Hash -> Seq Scan on plt1_adv_p2 t1_2 Filter: (b < 10) -> Hash Right Semi Join Hash Cond: ((t2_3.a = t1_3.a) AND (t2_3.c = t1_3.c)) -> Seq Scan on plt2_adv_p3_null t2_3 + Filter: (c IS NOT NULL) -> Hash -> Seq Scan on plt1_adv_p3 t1_3 Filter: (b < 10) -(21 rows) +(24 rows) SELECT t1.* FROM plt1_adv t1 WHERE EXISTS (SELECT 1 FROM plt2_adv t2 WHERE t1.a = t2.a AND t1.c = t2.c) AND t1.b < 10 ORDER BY t1.a; a | b | c diff --git a/src/test/regress/expected/subselect.out b/src/test/regress/expected/subselect.out index ce0ff764417..312da7ec662 100644 --- a/src/test/regress/expected/subselect.out +++ b/src/test/regress/expected/subselect.out @@ -401,14 +401,15 @@ select 1 = all (select (select 1)); explain (costs off) select * from int4_tbl o where exists (select 1 from int4_tbl i where i.f1=o.f1 limit null); - QUERY PLAN ------------------------------------- + QUERY PLAN +---------------------------------------- Hash Semi Join Hash Cond: (o.f1 = i.f1) -> Seq Scan on int4_tbl o -> Hash -> Seq Scan on int4_tbl i -(5 rows) + Filter: (f1 IS NOT NULL) +(6 rows) explain (costs off) select * from int4_tbl o where not exists @@ -764,8 +765,8 @@ explain (verbose, costs off) select * from semijoin_unique_tbl t1, semijoin_unique_tbl t2 where (t1.a, t2.a) in (select a+1, b+1 from semijoin_unique_tbl t3) order by t1.a, t2.a; - QUERY PLAN ------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------------- Incremental Sort Output: t1.a, t1.b, t2.a, t2.b Sort Key: t1.a, t2.a @@ -790,7 +791,8 @@ order by t1.a, t2.a; Group Key: (t3.a + 1), (t3.b + 1) -> Seq Scan on public.semijoin_unique_tbl t3 Output: t3.a, t3.b, (t3.a + 1), (t3.b + 1) -(24 rows) + Filter: (((t3.a + 1) IS NOT NULL) AND ((t3.b + 1) IS NOT NULL)) +(25 rows) -- encourage use of parallel plans set parallel_setup_cost=0; -- 2.45.1.windows.1