From 6ffc75a43c3c47c57b38f525c07272d2f4272357 Mon Sep 17 00:00:00 2001 From: Tomas Vondra Date: Fri, 17 Jul 2026 22:59:16 +0200 Subject: [PATCH v6 7/9] Properly plan filters built on joins The preceding commits allow filters to be built on joins on joins, but always estimate the selectivity based on just the 2-way join between the rel receiving the filter, and the immediate join partner. This may easily yield very inaccurate estimates, e.g. in a snowflake join with selective restrictions on the outer dimension tables. This commit enumerates possible joins by performing a significantly simplified version of standard_join_search. It merely constructs relids for the joins, without generating any paths etc. The relids are enough to estimate selectivity of filters built on the join, and generating the additional filtered paths. The enumeration (enumerate_bloom_filter_build_relids) runs once for the whole query, and the result is cached in PlannerInfo. There is a number of simplifications - it should not affect correctness of the final plan, in the worst case it generates joins that are later discared as invalid or does not generate some valid joins. The former can happen for semijoin unique-fication, approximated by checking semi_can_btree/semi_can_hash. The latter can happen for LATERAL joins, which simply disable join enumeration (for now). There are additional thresholds to limit the cost of join enumeration. We only generate joins up to 3 relations (BLOOM_MAX_BUILD_RELIDS) and stop after generating 100 (BLOOM_MAX_BUILD_SETS) joins. These are very rough thresholds, and likely will need more work. See enumerate_bloom_filter_build_relids for details, and a number of open questions in the comments. --- src/backend/optimizer/path/allpaths.c | 573 +++++++++++++++++++--- src/backend/optimizer/path/joinrels.c | 444 +++++++++++++++++ src/include/nodes/pathnodes.h | 13 + src/include/optimizer/paths.h | 1 + src/test/regress/expected/graph_table.out | 12 +- src/test/regress/expected/join.out | 4 +- 6 files changed, 976 insertions(+), 71 deletions(-) diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/path/allpaths.c index 937f14de526..160ab49d3c1 100644 --- a/src/backend/optimizer/path/allpaths.c +++ b/src/backend/optimizer/path/allpaths.c @@ -110,6 +110,10 @@ static void set_plain_rel_pathlist(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte); static List *find_interesting_bloom_filters(PlannerInfo *root, RelOptInfo *rel); +static Selectivity bloom_build_side_join_ratio(PlannerInfo *root, + RelOptInfo *rel, + Relids build_relids, + List **ec_all); static void generate_expected_filter_paths(PlannerInfo *root, RelOptInfo *rel); static void set_tablesample_rel_size(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte); @@ -1007,6 +1011,251 @@ bloom_filter_recipient_reachable(PlannerInfo *root, Index owner_relid, return true; } +/* + * bloom_build_side_join_ratio + * Estimate the fraction of owner (probe) rows that survive a Bloom filter + * built over the join relation 'build_relids'. + * + * The per-owner-clause semijoin estimate used elsewhere only looks at the + * clauses that directly join the owner to a build relation, using each build + * relation's base statistics. It therefore ignores selectivity contributed by + * the rest of the build side: restrictions on build relations (possibly ones + * that have no direct join clause to the owner at all) and the join clauses + * among the build relations. + * + * Consider a join on a snowflake schema with a fact table and two dimensions. + * + * SELECT * FROM fact_table + * JOIN dim_1 ON (fact_table.id1 = dim_1.id) + * JOIN dim_2 ON (dim_1.id2 = dim_2.id) + * WHERE dim_2.x = 10; + * + * The dimension table dim_2 is filtered by a WHERE clause and only reaches the + * fact table through another dimension table. The direct estimate is 1.0 (for + * the join between fact_table and dim_1), even though the join may eliminate + * almost every fact row (depending on how many tuples survive the WHERE). + * + * We approximate the surviving fraction as the estimated cardinality of the + * join of {owner} + build_relids divided by the owner's cardinality. Since a + * join emits at least one row per surviving owner row, this ratio is an upper + * bound on the true surviving fraction and so never makes the filter look more + * selective than it is. + * + * The join cardinality is the product of the build relations' (restriction + * reduced) row estimates times the selectivity of a non-redundant set of + * hashjoinable equality join clauses spanning the relation set. + * + * XXX We can't call generate_join_implied_equalities(), because all of this + * happens before we get to create the RelOptInfos for joins. So we assemble + * that clause set on our own. Only JOIN_INNER selectivity is used, so no + * join-relation lookup is needed either. + * + * XXX The RelOptInfo for baserels however already exist, and so we can use + * those without any issue. + * + * XXX What about RelOptInfos representing joins from previous join planning + * cycle (for queries above join_collapse_limit)? Those should also exist, + * no? We should still be able to access the baserels one by one. + */ +static Selectivity +bloom_build_side_join_ratio(PlannerInfo *root, RelOptInfo *rel, + Relids build_relids, List **ec_all) +{ + Relids setrelids = bms_union(rel->relids, build_relids); + List *clauses = NIL; + List *ec_clauses = NIL; + List *ecs = NIL; + int *dsu; + double nrows = 1.0; + Selectivity clausesel; + SpecialJoinInfo sjinfo; + ListCell *lc; + int i; + + /* + * Product of the build relations' restriction-reduced row estimates. That + * means it already reflects WHERE clauses on some of the rels (if any). + */ + i = -1; + while ((i = bms_next_member(build_relids, i)) >= 0) + { + RelOptInfo *br = root->simple_rel_array[i]; + + /* XXX paranoia */ + Assert(br != NULL); + + nrows *= br->rows; + } + + /* + * Collect candidate hashjoinable equality join clauses fully contained in + * the relation set. Ordinary (non-EC) join clauses are all applied; + * EC-derived clauses are gathered separately and later reduced to a + * non-redundant spanning set per EquivalenceClass. + * + * XXX I think this needs to be careful about using the RestrictInfo for + * the reason explained in find_interesting_bloom_filters, because we + * don't want to interfere with the regular selectivity estimation (by + * chaching our - possibly incorrect - estimate in the RestrictInfo). So + * we should strip the RestrictInfo, just like in + * find_interesting_bloom_filters, or make sure the estimate is correct + * (same as for regular planning). But we can strip that only after + * deduplicating the EC clauses etc. + */ + i = -1; + while ((i = bms_next_member(setrelids, i)) >= 0) + { + RelOptInfo *r = root->simple_rel_array[i]; + ListCell *lc2; + + /* XXX paranoia */ + Assert(r != NULL); + Assert(r->reloptkind == RELOPT_BASEREL); + + /* + * EC-derived clauses involving this rel (cached across build sides) + * + * XXX This caching is imperfect, in that 'empty list' is the same as + * 'not cached', so empty lists are recalculated over and over. Maybe + * we should split those two concepts, and remember when we got NIL? + * Unless calculating empty filter is very cheap, not sure. + */ + if (ec_all[i] == NIL) + ec_all[i] = generate_implied_equalities_for_all_columns(root, r, + bloom_em_matches_anybarevar, + NULL, NULL); + foreach(lc2, ec_all[i]) + { + RestrictInfo *rinfo = (RestrictInfo *) lfirst(lc2); + + if (rinfo->parent_ec == NULL) + continue; + + if (bms_is_subset(rinfo->clause_relids, setrelids)) + { + ec_clauses = list_append_unique_ptr(ec_clauses, rinfo); + ecs = list_append_unique_ptr(ecs, rinfo->parent_ec); + } + } + + /* Ordinary (non-EC) join clauses involving this rel. */ + foreach(lc2, r->joininfo) + { + RestrictInfo *rinfo = (RestrictInfo *) lfirst(lc2); + + if (rinfo->parent_ec != NULL) + continue; + + /* XXX Can this actually happen for join clauses? */ + if (bms_membership(rinfo->clause_relids) != BMS_MULTIPLE) + continue; + + if (bms_is_subset(rinfo->clause_relids, setrelids)) + clauses = list_append_unique_ptr(clauses, rinfo); + } + } + + /* + * Reduce the EC-derived clauses to a non-redundant spanning set. + * + * Within each EquivalenceClass, several of the collected clauses may + * enforce the same equality transitively (e.g. a=b, a=c and b=c when + * three relations of the set share a class); keeping them all would + * multiply the same selectivity more than once. + * + * We use a small union-find (aka DSU) over the relids of the set, reset + * per class, and keep a clause only when it connects two so-far + * unconnected relations. + * + * The DSU tracks "parent" representative for each entry. We start with + * each entry being it's own parent, and gradually merge disjoint sets + * connected by the EC clauses. + * + * see https://en.wikipedia.org/wiki/Disjoint-set_data_structure + */ + dsu = palloc_array(int, root->simple_rel_array_size); + foreach(lc, ecs) + { + EquivalenceClass *ec = (EquivalenceClass *) lfirst(lc); + ListCell *lc2; + + /* reset the DSU, each relation is a separate group */ + for (i = 0; i < root->simple_rel_array_size; i++) + dsu[i] = i; + + foreach(lc2, ec_clauses) + { + RestrictInfo *rinfo = (RestrictInfo *) lfirst(lc2); + int first = -1; + bool merged = false; + int m; + + /* skip clauses that don't belong to this EC */ + if (rinfo->parent_ec != ec) + continue; + + /* Union all set members referenced by this clause. */ + m = -1; + while ((m = bms_next_member(rinfo->clause_relids, m)) >= 0) + { + int ra, + rb; + + Assert(m < root->simple_rel_array_size); + + /* first rel in this EC clause */ + if (first < 0) + { + first = m; + continue; + } + + /* find (first) - representative of the first rel */ + for (ra = first; dsu[ra] != ra; ra = dsu[ra]) + /* nothing */ ; + + /* find (m) - representative of the current rel */ + for (rb = m; dsu[rb] != rb; rb = dsu[rb]) + /* nothing */ ; + + /* different representatives = disjoint sets, merge them */ + if (ra != rb) + { + dsu[rb] = ra; + merged = true; + } + } + + /* + * If the clause connected two previously disconnected sets of + * relations, it's not redundant. So use it for estimating filter + * selectivity. + */ + if (merged) + clauses = lappend(clauses, rinfo); + } + } + pfree(dsu); + + /* no clauses, no filtering */ + if (clauses == NIL) + { + bms_free(setrelids); + return 1.0; + } + + /* + * XXX find_interesting_bloom_filters uses JOIN_SEMI, so maybe we should + * use the same thing here? + */ + init_dummy_sjinfo(&sjinfo, rel->relids, build_relids); + clausesel = clauselist_selectivity(root, clauses, 0, JOIN_INNER, &sjinfo); + + bms_free(setrelids); + + return nrows * clausesel; +} + /* * find_interesting_bloom_filters * Identify Bloom filters that a hash join above this scan could push down, @@ -1037,9 +1286,14 @@ static List * find_interesting_bloom_filters(PlannerInfo *root, RelOptInfo *rel) { List *candidates; - List *group_relids = NIL; /* parallel: Relids per group */ - List *group_clauses = NIL; /* parallel: List of clauses */ + List **clauses_by_rel; /* clauses per build relations */ + double *sel_by_rel; /* selectivity per build relation */ + bool *has_rel; /* XXX redundant with (clauses_by_rel[r] != + * NIL) */ + List **ec_all; + List *build_sides; List *result = NIL; + int i; ListCell *lc; if (!enable_hashjoin_bloom) @@ -1067,7 +1321,27 @@ find_interesting_bloom_filters(PlannerInfo *root, RelOptInfo *rel) candidates = lappend(candidates, rinfo->clause); } - /* Group candidate clauses by their build-side relids. */ + /* + * Group usable candidate clauses by the base relation on the build side. + * + * A clause is usable only if its owner side is a bare Var of this + * relation and its build side is a single base relation; that keeps the + * per-relation selectivity estimate below well-defined (a semijoin + * selectivity with a multi-relation inner side would require a join + * RelOptInfo that does not exist yet at this stage of planning). + * + * A build side spanning several relations is still supported, but it is + * assembled from several such single-relation clauses (see below), not + * from a single clause referencing many relations. + * + * XXX Is the comment about not having join RelOptInfo accurate, even with + * the enumerate_bloom_filter_build_relids below? Couldn't we use that to + * calculate the correct semijoin selectivity using that, even for clauses + * referencing multiple relations? But it's fairly rare case. + */ + clauses_by_rel = palloc0_array(List *, root->simple_rel_array_size); + has_rel = palloc0_array(bool, root->simple_rel_array_size); + foreach(lc, candidates) { Node *clause = (Node *) lfirst(lc); @@ -1078,9 +1352,6 @@ find_interesting_bloom_filters(PlannerInfo *root, RelOptInfo *rel) Relids clause_relids; Relids build_relids; int buildrel; - ListCell *lc2; - ListCell *lc3; - bool found; /* strip RestrictInfo (see comment above) */ if (IsA(clause, RestrictInfo)) @@ -1119,95 +1390,243 @@ find_interesting_bloom_filters(PlannerInfo *root, RelOptInfo *rel) if (!op_hashjoinable(opno, exprType(ownerexpr))) continue; - /* - * The build side must be a single base relation; that's what the - * recipient lookup and our selectivity estimate can handle. - * - * XXX I don't think this restriction is necessary. We can allow the - * build side to be a join. I don't see why that would be a problem. - */ + /* The build side of a usable clause must be one base relation. */ clause_relids = pull_varnos(root, (Node *) clause); build_relids = bms_difference(clause_relids, rel->relids); if (!bms_get_singleton_member(build_relids, &buildrel) || - buildrel >= root->simple_rel_array_size || root->simple_rel_array[buildrel] == NULL || root->simple_rel_array[buildrel]->reloptkind != RELOPT_BASEREL) { bms_free(build_relids); continue; } + bms_free(build_relids); - /* Add to an existing group, or start a new one. */ + Assert(root->simple_rel_array[buildrel]->reloptkind == RELOPT_BASEREL); - /* - * XXX Maybe we sould have a HTAB with the relids as a key? But the - * lists should not be that long, I think. - */ - found = false; - forboth(lc2, group_relids, lc3, group_clauses) - { - Relids grelids = (Relids) lfirst(lc2); + clauses_by_rel[buildrel] = lappend(clauses_by_rel[buildrel], clause); + has_rel[buildrel] = true; + } - if (bms_equal(grelids, build_relids)) - { - lfirst(lc3) = lappend((List *) lfirst(lc3), clause); - found = true; + /* + * Pre-compute the per-build-relation semijoin selectivity. + * + * XXX I think we could calculate the selectivity only for semijoin-like + * joins (semijoin, inner join), and ignore the rest. We don't even need + * to calculate interesting filters for those cases. + */ + sel_by_rel = palloc_array(double, root->simple_rel_array_size); + for (i = 0; i < root->simple_rel_array_size; i++) + { + SpecialJoinInfo sjinfo; + Relids build_relids; - /* added to an existing group, don't keep the relids around */ - bms_free(build_relids); + /* ignore relations without join clauses */ + if (!has_rel[i]) + continue; - break; - } - } + build_relids = bms_make_singleton(i); + init_dummy_sjinfo(&sjinfo, rel->relids, build_relids); + sjinfo.jointype = JOIN_SEMI; - if (!found) /* new group */ - { - group_relids = lappend(group_relids, build_relids); - group_clauses = lappend(group_clauses, list_make1(clause)); - } + sel_by_rel[i] = clauselist_selectivity(root, clauses_by_rel[i], 0, + JOIN_SEMI, &sjinfo); + bms_free(build_relids); } /* - * We have collected all potentially intresting filters. Evaluate - * selectivity of each group and keep only the most interesting filters. - * Filters have to eliminate at least bloom_filter_pushdown_threshold - * tuples, and we keep only bloom_filter_pushdown_max most selective ones. + * Generate the set of possible join relations on the build side. + * + * This performs a minimal subset of the join planning - construct joins + * as set of relids, for which we can build filters. We don't bother with + * generating paths or even constructing RelOptInfos for joins, all that + * will happen later during the "full" join search. We only care about + * which joins would be valid, and calculating the selectivity. + * + * There are various other limits and heuristics intended to minimize the + * cost, see enumerate_bloom_filter_build_relids for details. + * + * The list of valid join relids is calculated once, on the first call, + * and cached in PlannerInfo. + * + * XXX We could invent additional heuristics for when to calculate the + * join relids, so that we don't do that in cases where it's obviously + * useless (for queries touching only tiny amounts of rows, ...). We + * should also give CustomScan to decide whether to do this (so that it + * can request it in more cases, if it can leverage it in some way, e.g. + * to scan storage more efficiently). + */ + build_sides = enumerate_bloom_filter_build_relids(root); + + /* Cache of per-relation EC-derived clauses, filled lazily below. */ + ec_all = palloc0_array(List *, root->simple_rel_array_size); + + /* + * Consider each legal join relation (or single base relation) that could + * serve as a build side. For a build side B we combine the per-relation + * clauses of every build relation in B that has a direct join clause to + * the owner; those clauses define the filter keys. At least one such + * relation must exist, otherwise the filter would have no keys. + * + * We estimate the surviving fraction two ways and keep the smaller (more + * selective) one, since both are upper bounds on the true value: + * + * - the product of the per-relation semijoin selectivities (assuming + * independence), which uses each build relation's base statistics but + * ignores the rest of the build side; and + * + * - the build-side join-cardinality ratio (see + * bloom_build_side_join_ratio), which accounts for restrictions on build + * relations and the join clauses among them -- including build relations + * with no direct join clause to the owner. + * + * The whole connected set B is used as the filter's build side, so the + * filter is sourced by the join that first brings all of B together with + * the owner (after any restriction on B's members has been applied). + * + * XXX Isn't there a better way to estimate the selectivity of the filter? + * The two alternative formulas seem a bit strange / suspicious. Shouldn't + * we really multiply all of this together, somehow? */ + foreach(lc, build_sides) { - ListCell *lcr = list_head(group_relids); - ListCell *lcc = list_head(group_clauses); + Relids bset = (Relids) lfirst(lc); /* relids on build side */ + List *clauses = NIL; + Selectivity sel = 1.0; + ListCell *lce; + bool add; + int r; + + /* The owner cannot be on the build side of the filter. */ + if (bms_is_member(rel->relid, bset)) + continue; - while (lcr != NULL && lcc != NULL) + /* + * Collect the owner-side clauses (the filter keys) and the product of + * the per-relation semijoin selectivities. + */ + r = -1; + while ((r = bms_next_member(bset, r)) >= 0) { - Relids build_relids = (Relids) lfirst(lcr); - List *clauses = (List *) lfirst(lcc); - SpecialJoinInfo sjinfo; - Selectivity sel; + if (!has_rel[r]) + continue; - init_dummy_sjinfo(&sjinfo, rel->relids, build_relids); - sjinfo.jointype = JOIN_SEMI; + clauses = list_concat(clauses, list_copy(clauses_by_rel[r])); + sel *= sel_by_rel[r]; + } + + /* + * Have a clause to the filter "owner" (where we push the filter) for + * this set of build relids? If not, it's a crossproduct, and that + * can't filter anything. + */ + if (clauses == NIL) + continue; - sel = clauselist_selectivity(root, clauses, 0, JOIN_SEMI, &sjinfo); + /* + * For a build side that spans several relations, refine the estimate + * with the build-side join-cardinality ratio and keep whichever bound + * is tighter. This is what accounts for restrictions on build + * relations and for join clauses among them -- including build + * relations that have no direct join clause to the owner. A + * single-relation build side has no such internal structure, so the + * per-relation semijoin estimate already covers it. + */ + if (bms_membership(bset) == BMS_MULTIPLE) + { + Selectivity join_ratio; - if ((sel <= 1.0 - bloom_filter_pushdown_threshold) && - (sel > 0.0) && /* XXX seems unnecessary */ - bloom_filter_recipient_reachable(root, rel->relid, build_relids)) - { - ExpectedFilter *f = makeNode(ExpectedFilter); + join_ratio = bloom_build_side_join_ratio(root, rel, bset, ec_all); - f->owner_relid = rel->relid; - f->build_relids = build_relids; - f->clauses = clauses; - f->selectivity = sel; - result = lappend(result, f); + if (join_ratio < sel) + sel = join_ratio; + } + + Assert((sel >= 0.0) && (sel <= 1.0)); + + /* + * A candidate we would not emit anyway (not selective enough, or not + * reachable at the owner's scan) must not influence the filters we + * have already collected, so test those conditions before the + * de-duplication below. + */ + if (!(sel <= 1.0 - bloom_filter_pushdown_threshold && sel > 0.0) || + !bloom_filter_recipient_reachable(root, rel->relid, bset)) + { + list_free(clauses); + continue; + } + + /* + * De-duplicate against the filters produced so far. Beyond dropping + * an exact duplicate build side, we keep a larger filter only when it + * is strictly more selective (discards more tuples) than a smaller + * one it contains: a smaller build side is sourced no later and costs + * no more, so it dominates a larger build side that does not discard + * more. + * + * - If an existing filter has the same build side, keep the existing + * filter and drop this candidate. + * + * - If an existing filter's build side is a subset of this one (the + * candidate is larger), keep the candidate only if it is strictly + * more selective than that existing filter. + * + * - If an existing filter's build side is a superset of this one (the + * candidate is smaller), drop that existing filter unless it is + * strictly more selective than the candidate. + * + * XXX Can we actually see the same set of build relids twice? I don't + * think that should be possible, we only generate a single list, and + * that should not contain duplicate relids. + */ + add = true; + foreach(lce, result) + { + ExpectedFilter *f = (ExpectedFilter *) lfirst(lce); + + if (bms_equal(f->build_relids, bset)) + { + add = false; + break; + } + else if (bms_is_subset(f->build_relids, bset)) + { + /* existing (smaller) dominates unless candidate is better */ + if (f->selectivity <= sel) + { + add = false; + break; + } } + else if (bms_is_subset(bset, f->build_relids)) + { + /* candidate (smaller) dominates the existing larger filter */ + if (sel <= f->selectivity) + result = foreach_delete_current(result, lce); + } + } - lcr = lnext(group_relids, lcr); - lcc = lnext(group_clauses, lcc); + if (add) + { + ExpectedFilter *f = makeNode(ExpectedFilter); + + f->owner_relid = rel->relid; + f->build_relids = bms_copy(bset); + f->clauses = clauses; + f->selectivity = sel; + result = lappend(result, f); } + else + list_free(clauses); } /* + * We have collected all potentially intresting filters. Evaluate + * selectivity of each group and keep only the most interesting filters. + * Filters have to eliminate at least bloom_filter_pushdown_threshold + * tuples, and we keep only bloom_filter_pushdown_max most selective ones. + * * We only connsider a limited number of interesting filters, to prevent * path explosion. If we found too many, keep only the most selective ones * (with smallest surviving fraction of tuples), to bound the number of @@ -1345,6 +1764,8 @@ generate_expected_filter_paths(PlannerInfo *root, RelOptInfo *rel) for (combo = 1; combo < ((uint32) 1 << nfilters); combo++) { List *subset = NIL; + Relids used = NULL; + bool overlap = false; int i = 0; ListCell *lcf; @@ -1355,6 +1776,32 @@ generate_expected_filter_paths(PlannerInfo *root, RelOptInfo *rel) i++; } + /* + * Skip combinations that mix filters with overlapping build sides. + * Such filters would be sourced from joins that share build + * relations, so their selectivities are not independent and applying + * them together at the same scan wrong - the result would be correct, + * but the estimates would get smaller. + */ + foreach(lcf, subset) + { + ExpectedFilter *f = (ExpectedFilter *) lfirst(lcf); + + if (bms_overlap(used, f->build_relids)) + { + overlap = true; + break; + } + used = bms_add_members(used, f->build_relids); + } + bms_free(used); + + if (overlap) + { + list_free(subset); + continue; + } + /* * All filtered paths for this combo share the same expected_filters * list. That's safe: the list is never modified, and add_path() only diff --git a/src/backend/optimizer/path/joinrels.c b/src/backend/optimizer/path/joinrels.c index 443e2dca7c0..c73f3e69a7c 100644 --- a/src/backend/optimizer/path/joinrels.c +++ b/src/backend/optimizer/path/joinrels.c @@ -649,6 +649,450 @@ join_is_legal(PlannerInfo *root, RelOptInfo *rel1, RelOptInfo *rel2, return true; } +/* + * simple_join_is_legal + * Relids-only variant of join_is_legal(), used by the Bloom-filter + * build-side enumeration to decide whether a set of base relations can + * legally be joined into a single join relation, without building any + * RelOptInfo or generating paths. + * + * This mirrors the special-join legality logic of join_is_legal(), operating + * purely on relid sets. Two simplifications are made relative to the real + * function: + * + * - Semijoin unique-ification, which join_is_legal() probes via + * create_unique_paths(), is approximated by the sjinfo's semi_can_btree / + * semi_can_hash flags (create_unique_paths() ultimately succeeds only when + * one of those holds; see planner.c). + * + * - LATERAL handling is omitted; callers must not use this on queries with + * lateral references (see enumerate_bloom_filter_build_relids, which skips + * the whole feature when hasLateralRTEs is set). + * + * The result is purely advisory: it is only used to decide which build sides + * to consider for Bloom filters. Any filter that turns out not to be + * realizable is later rejected by compute_join_expected_filters(), so it is + * safe for this test to be conservative (an over-permissive or over-strict + * answer only affects planning effort, never correctness). + */ +static bool +simple_join_is_legal(PlannerInfo *root, Relids relids1, Relids relids2, + Relids joinrelids) +{ + SpecialJoinInfo *match_sjinfo; + bool must_be_leftjoin; + ListCell *l; + + Assert(!root->hasLateralRTEs); + + match_sjinfo = NULL; + must_be_leftjoin = false; + + foreach(l, root->join_info_list) + { + SpecialJoinInfo *sjinfo = (SpecialJoinInfo *) lfirst(l); + + /* + * This special join is not relevant unless its RHS overlaps the + * proposed join. (Check this first as a fast path for dismissing + * most irrelevant SJs quickly.) + */ + if (!bms_overlap(sjinfo->min_righthand, joinrelids)) + continue; + + /* + * Also, not relevant if proposed join is fully contained within RHS + * (ie, we're still building up the RHS). + */ + if (bms_is_subset(joinrelids, sjinfo->min_righthand)) + continue; + + /* + * Also, not relevant if SJ is already done within either input. + */ + if (bms_is_subset(sjinfo->min_lefthand, relids1) && + bms_is_subset(sjinfo->min_righthand, relids1)) + continue; + if (bms_is_subset(sjinfo->min_lefthand, relids2) && + bms_is_subset(sjinfo->min_righthand, relids2)) + continue; + + /* + * If it's a semijoin and we already joined the RHS to any other rels + * within either input, then we must have unique-ified the RHS at that + * point (see below). Therefore the semijoin is no longer relevant in + * this join path. + */ + if (sjinfo->jointype == JOIN_SEMI) + { + if (bms_is_subset(sjinfo->syn_righthand, relids1) && + !bms_equal(sjinfo->syn_righthand, relids1)) + continue; + if (bms_is_subset(sjinfo->syn_righthand, relids2) && + !bms_equal(sjinfo->syn_righthand, relids2)) + continue; + } + + /* + * If one input contains min_lefthand and the other contains + * min_righthand, then we can perform the SJ at this join. + * + * Reject if we get matches to more than one SJ; that implies we're + * considering something that's not really valid. + */ + if (bms_is_subset(sjinfo->min_lefthand, relids1) && + bms_is_subset(sjinfo->min_righthand, relids2)) + { + if (match_sjinfo) + return false; /* invalid join path */ + match_sjinfo = sjinfo; + /* reversed = false; */ + } + else if (bms_is_subset(sjinfo->min_lefthand, relids2) && + bms_is_subset(sjinfo->min_righthand, relids1)) + { + if (match_sjinfo) + return false; /* invalid join path */ + match_sjinfo = sjinfo; + /* reversed = true; */ + } + else if (sjinfo->jointype == JOIN_SEMI && + bms_equal(sjinfo->syn_righthand, relids2) && + (sjinfo->semi_can_btree || sjinfo->semi_can_hash)) /* XXX + * create_unique_paths */ + { + /* Unique-ified semijoin RHS (see join_is_legal). */ + /*---------- + * For a semijoin, we can join the RHS to anything else by + * unique-ifying the RHS (if the RHS can be unique-ified). + * We will only get here if we have the full RHS but less + * than min_lefthand on the LHS. + * + * The reason to consider such a join path is exemplified by + * SELECT ... FROM a,b WHERE (a.x,b.y) IN (SELECT c1,c2 FROM c) + * If we insist on doing this as a semijoin we will first have + * to form the cartesian product of A*B. But if we unique-ify + * C then the semijoin becomes a plain innerjoin and we can join + * in any order, eg C to A and then to B. When C is much smaller + * than A and B this can be a huge win. So we allow C to be + * joined to just A or just B here, and then make_join_rel has + * to handle the case properly. + * + * Note that actually we'll allow unique-ified C to be joined to + * some other relation D here, too. That is legal, if usually not + * very sane, and this routine is only concerned with legality not + * with whether the join is good strategy. + *---------- + */ + if (match_sjinfo) + return false; /* invalid join path */ + match_sjinfo = sjinfo; + } + else if (sjinfo->jointype == JOIN_SEMI && + bms_equal(sjinfo->syn_righthand, relids1) && + (sjinfo->semi_can_btree || sjinfo->semi_can_hash)) /* XXX + * create_unique_paths */ + { + /* Reversed unique-ified semijoin case. */ + if (match_sjinfo) + return false; /* invalid join path */ + match_sjinfo = sjinfo; + } + else + { + /* See the corresponding branch in join_is_legal. */ + /* + * Otherwise, the proposed join overlaps the RHS but isn't a valid + * implementation of this SJ. But don't panic quite yet: the RHS + * violation might have occurred previously, in one or both input + * relations, in which case we must have previously decided that + * it was OK to commute some other SJ with this one. If we need + * to perform this join to finish building up the RHS, rejecting + * it could lead to not finding any plan at all. (This can occur + * because of the heuristics elsewhere in this file that postpone + * clauseless joins: we might not consider doing a clauseless join + * within the RHS until after we've performed other, validly + * commutable SJs with one or both sides of the clauseless join.) + * This consideration boils down to the rule that if both inputs + * overlap the RHS, we can allow the join --- they are either + * fully within the RHS, or represent previously-allowed joins to + * rels outside it. + */ + if (bms_overlap(relids1, sjinfo->min_righthand) && + bms_overlap(relids2, sjinfo->min_righthand)) + continue; /* assume valid previous violation of RHS */ + + /* + * The proposed join could still be legal, but only if we're + * allowed to associate it into the RHS of this SJ. That means + * this SJ must be a LEFT join (not SEMI or ANTI, and certainly + * not FULL) and the proposed join must not overlap the LHS. + */ + if (sjinfo->jointype != JOIN_LEFT || + bms_overlap(joinrelids, sjinfo->min_lefthand)) + return false; /* invalid join path */ + + /* + * To be valid, the proposed join must be a LEFT join; otherwise + * it can't associate into this SJ's RHS. But we may not yet have + * found the SpecialJoinInfo matching the proposed join, so we + * can't test that yet. Remember the requirement for later. + */ + must_be_leftjoin = true; + } + } + + /* + * Fail if we violated some SJ's RHS and didn't match to a strict LEFT SJ. + */ + if (must_be_leftjoin && + (match_sjinfo == NULL || + match_sjinfo->jointype != JOIN_LEFT || + !match_sjinfo->lhs_strict)) + return false; /* invalid join path */ + + /* Otherwise, it's a valid join */ + return true; +} + +/* + * simple_have_relevant_joinclause + * Is there a join clause (or join-order restriction) linking any base + * relation in relids1 to any base relation in relids2? + * + * This is a relids-level connectivity test built on top of the per-baserel + * have_relevant_joinclause() / have_join_order_restriction(), used by the + * Bloom-filter build-side enumeration. It intentionally works pairwise over + * base relations, which may miss clauses that only become relevant once three + * or more relations are joined together; that merely causes the enumeration to + * under-generate candidate build sides, which is safe. + */ +static bool +simple_have_relevant_joinclause(PlannerInfo *root, Relids relids1, + Relids relids2) +{ + int r1; + + r1 = -1; + while ((r1 = bms_next_member(relids1, r1)) >= 0) + { + RelOptInfo *rel1 = root->simple_rel_array[r1]; + int r2; + + if (rel1 == NULL) + continue; + + r2 = -1; + while ((r2 = bms_next_member(relids2, r2)) >= 0) + { + RelOptInfo *rel2 = root->simple_rel_array[r2]; + + if (rel2 == NULL) + continue; + + if (have_relevant_joinclause(root, rel1, rel2) || + have_join_order_restriction(root, rel1, rel2)) + return true; + } + } + + return false; +} + +/* + * Maximum number of base relations allowed in an enumerated Bloom-filter + * build side. Bloom filters over larger joins are unlikely to be worthwhile + * and enumerating them would inflate planning time, so we keep this small. + * + * This bounds the *size* of each candidate build side, which is distinct from + * the bloom_filter_pushdown_max GUC that bounds how many interesting filters + * are ultimately kept per probe relation. + */ +#define BLOOM_MAX_BUILD_RELIDS 3 + +/* + * Overall cap on the number of enumerated build-side relid sets, as a safety + * valve against pathological join graphs. + */ +#define BLOOM_MAX_BUILD_SETS 100 + +/* + * enumerate_bloom_filter_build_relids + * Enumerate the legal join relations (and single base relations) that + * could serve as the build side of a pushed-down hash-join Bloom filter. + * + * The result is a list of Relids. Level-1 entries are the individual base + * relations; higher levels are unions that form a legal join relation and are + * connected by at least one join clause or join-order restriction. This is a + * simplified, relids-only analogue of join_search_one_level() that builds no + * RelOptInfo and generates no paths. + * + * The list is cached in the PlannerInfo, since it is global to the query and + * is consulted once per base relation by find_interesting_bloom_filters(). + * + * Returns NIL (and does no work) when the feature is disabled, when there are + * lateral references, or when the join problem is large enough that GEQO would + * be used, in order to bound planning cost. + * + * This is a simplified variant of the join search, and the generated list of + * join relids may be incomplete for several reasons. We don't even try to + * generate joins of more than BLOOM_MAX_BUILD_RELIDS baserels; in snowflake + * schemas this limits how many dimensions may push filter to the fact table. + * We also don't try to generate more than BLOOM_MAX_BUILD_SETS relids sets + * (this serves as a "budget"). + * + * Furthermore, simple_have_relevant_joinclause considers only joins over + * pair-wise clauses. This will skip complex join clauses, i.e. clauses + * referencing three or more relations. + * + * XXX Should this use a different data structure other than a linked list? + * Preferably one that would allow (a) cheap duplicate detection, and (b) + * filtering by relid of a particular relation (joins it participates in). + * The list works well enough for small BLOOM_MAX_BUILD_SETS values, but + * with larger lists it might be expensive. + * + * XXX I think it might be possible to "prune" the enumerated joins by first + * checking which relations are worth receiving a pushed-down filter, and + * then only "building" filters for those "seeds". It does not matter if we + * join (F,D1,D2) or (F,(D1,D2)), what matters is the final relids set. So + * we could "add" the rels one by one to the fact table (and then we'd need + * to subtract it, to get the build side). Complex join clauses would break + * this logic (can't join tables one by one), but that case is not very + * common, I think. Seems like an acceptable trade off. + * + * XXX Could we "explore" the join graph a bit, and use that to determine + * interesting filters? Starjoin/snowflake joins should have fairly typical + * pattern, I think. Could help to "focus" which joins to enumerate. We + * might also calculate "join depth" for each "interesting" fact table, + * i.e. the maximum length of a join chain, or something. Could help as a + * heuristic to limit the enumeration. + */ +List * +enumerate_bloom_filter_build_relids(PlannerInfo *root) +{ + List *levels[BLOOM_MAX_BUILD_RELIDS + 1]; + List *result = NIL; + int nbaserels = 0; + int level; + int i; + + /* bail out if filter pushdown disabled */ + if (!enable_hashjoin_bloom || bloom_filter_pushdown_max <= 0) + return NIL; + + /* Return the cached answer if we've already computed it. */ + if (root->bloom_build_relids_valid) + return root->bloom_build_relids; + + root->bloom_build_relids_valid = true; + root->bloom_build_relids = NIL; + + /* Level 1: each base relation on its own. */ + levels[1] = NIL; + for (i = 1; i < root->simple_rel_array_size; i++) + { + RelOptInfo *rel = root->simple_rel_array[i]; + + if (rel == NULL || rel->reloptkind != RELOPT_BASEREL) + continue; + + nbaserels++; + levels[1] = lappend(levels[1], bms_make_singleton(i)); + } + + /* + * Skip multi-rel enumeration for trivial or very large join problems, and + * when there are LATERAL references (which complicate join legality in + * ways simple_join_is_legal does not model yets). + * + * We still expose the level-1 singletons, so single-base-rel build sides + * keep working as before; we simply don't combine them into join + * relations. + * + * For large joins we would (a) spend too much effort here and (b) likely + * fall back to GEQO, whose plans this speculative work would not help. + */ + if (nbaserels < 2 || + root->hasLateralRTEs || + (enable_geqo && nbaserels >= geqo_threshold)) + { + root->bloom_build_relids = levels[1]; + return root->bloom_build_relids; + } + + result = list_copy(levels[1]); + + /* Build higher levels by adding one base relation at a time. */ + for (level = 2; level <= BLOOM_MAX_BUILD_RELIDS; level++) + { + ListCell *lc; + + levels[level] = NIL; + + foreach(lc, levels[level - 1]) + { + Relids oldrelids = (Relids) lfirst(lc); + ListCell *lc2; + + foreach(lc2, levels[1]) + { + Relids singleton = (Relids) lfirst(lc2); + int addrel = bms_singleton_member(singleton); + Relids joinrelids; + ListCell *lc3; + bool dup; + + /* The added rel must not already be part of the set. */ + if (bms_is_member(addrel, oldrelids)) + continue; + + /* Must be connected by a join clause or order restriction. */ + if (!simple_have_relevant_joinclause(root, oldrelids, + singleton)) + continue; + + joinrelids = bms_union(oldrelids, singleton); + + /* Skip if we already produced this set at this level. */ + dup = false; + foreach(lc3, levels[level]) + { + if (bms_equal((Relids) lfirst(lc3), joinrelids)) + { + dup = true; + break; + } + } + if (dup) + { + bms_free(joinrelids); + continue; + } + + /* Must form a legal join relation. */ + if (!simple_join_is_legal(root, oldrelids, singleton, + joinrelids)) + { + bms_free(joinrelids); + continue; + } + + levels[level] = lappend(levels[level], joinrelids); + result = lappend(result, joinrelids); + + if (list_length(result) >= BLOOM_MAX_BUILD_SETS) + { + root->bloom_build_relids = result; + return result; + } + } + } + } + + root->bloom_build_relids = result; + return result; +} + /* * init_dummy_sjinfo * Populate the given SpecialJoinInfo for a plain inner join between the diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index 2dbbc241420..cd3e0128f4f 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -693,6 +693,19 @@ struct PlannerInfo /* PartitionPruneInfos added in this query's plan. */ List *partPruneInfos; + /* + * Cache for Bloom-filter pushdown build-side enumeration (see + * find_interesting_bloom_filters in allpaths.c and + * enumerate_bloom_filter_build_relids in joinrels.c). This is a list of + * Relids, each identifying a legal join relation (or single base + * relation) that could serve as the build side of a pushed-down hash-join + * Bloom filter. Computed lazily and reused across all base relations, so + * a separate "valid" flag distinguishes "not yet computed" from + * "computed, no candidates". + */ + List *bloom_build_relids pg_node_attr(read_write_ignore); + bool bloom_build_relids_valid pg_node_attr(read_write_ignore); + /* extension state */ void **extension_state pg_node_attr(read_write_ignore); int extension_state_allocated; diff --git a/src/include/optimizer/paths.h b/src/include/optimizer/paths.h index 636761f2e94..59801f9b0df 100644 --- a/src/include/optimizer/paths.h +++ b/src/include/optimizer/paths.h @@ -118,6 +118,7 @@ extern Relids add_outer_joins_to_relids(PlannerInfo *root, Relids input_relids, List **pushed_down_joins); extern bool have_join_order_restriction(PlannerInfo *root, RelOptInfo *rel1, RelOptInfo *rel2); +extern List *enumerate_bloom_filter_build_relids(PlannerInfo *root); extern void mark_dummy_rel(RelOptInfo *rel); extern void init_dummy_sjinfo(SpecialJoinInfo *sjinfo, Relids left_relids, Relids right_relids); diff --git a/src/test/regress/expected/graph_table.out b/src/test/regress/expected/graph_table.out index 6241b189ed8..553fe8e5c0c 100644 --- a/src/test/regress/expected/graph_table.out +++ b/src/test/regress/expected/graph_table.out @@ -423,8 +423,8 @@ SELECT * FROM GRAPH_TABLE (g1 MATCH (a IS vl1 | vl2) COLUMNS (a.vname, a.vprop1) SELECT src, conn, dest, lprop1, vprop2, vprop1 FROM GRAPH_TABLE (g1 MATCH (a IS vl1)-[b IS el1]->(c IS vl2 | vl3) COLUMNS (a.vname AS src, b.ename AS conn, c.vname AS dest, c.lprop1, c.vprop2, c.vprop1)); src | conn | dest | lprop1 | vprop2 | vprop1 -----+------+------+----------+--------+-------- - v11 | e121 | v22 | vl2_prop | 1200 | 1020 v12 | e122 | v21 | vl2_prop | 1100 | 1010 + v11 | e121 | v22 | vl2_prop | 1200 | 1020 v11 | e131 | v33 | vl3_prop | | 2030 v11 | e132 | v31 | vl3_prop | | 2010 (4 rows) @@ -433,16 +433,16 @@ SELECT src, conn, dest, lprop1, vprop2, vprop1 FROM GRAPH_TABLE (g1 MATCH (a IS SELECT * FROM GRAPH_TABLE (g1 MATCH (v1 IS vl2)-[conn]-(v2) COLUMNS (v1.vname AS v1name, conn.ename AS cname, v2.vname AS v2name)); v1name | cname | v2name --------+-------+-------- - v22 | e121 | v11 v21 | e122 | v12 + v22 | e121 | v11 v22 | e231 | v32 (3 rows) SELECT * FROM GRAPH_TABLE (g1 MATCH (v1 IS vl2)-(v2) COLUMNS (v1.vname AS v1name, v2.vname AS v2name)); v1name | v2name --------+-------- - v22 | v11 v21 | v12 + v22 | v11 v22 | v32 (3 rows) @@ -503,8 +503,8 @@ LINE 1: SELECT * FROM GRAPH_TABLE (g1 MATCH (WHERE b.eprop1 = 10001)... SELECT * FROM GRAPH_TABLE (g1 MATCH (src)-[conn]->(dest) COLUMNS (src.vname AS svname, conn.ename AS cename, dest.vname AS dvname, src.vprop1 AS svp1, src.vprop2 AS svp2, src.lprop1 AS slp1, dest.vprop1 AS dvp1, dest.vprop2 AS dvp2, dest.lprop1 AS dlp1, conn.eprop1 AS cep1, conn.lprop2 AS clp2)); svname | cename | dvname | svp1 | svp2 | slp1 | dvp1 | dvp2 | dlp1 | cep1 | clp2 --------+--------+--------+------+------+----------+------+------+----------+-------+-------- - v11 | e121 | v22 | 10 | | | 1020 | 1200 | vl2_prop | 10001 | v12 | e122 | v21 | 20 | | | 1010 | 1100 | vl2_prop | 10002 | + v11 | e121 | v22 | 10 | | | 1020 | 1200 | vl2_prop | 10001 | v11 | e131 | v33 | 10 | | | 2030 | | vl3_prop | 10003 | v11 | e132 | v31 | 10 | | | 2010 | | vl3_prop | 10004 | v22 | e231 | v32 | 1020 | 1200 | vl2_prop | 2020 | | vl3_prop | | 100050 @@ -514,8 +514,8 @@ SELECT * FROM GRAPH_TABLE (g1 MATCH (src)-[conn]->(dest) COLUMNS (src.vname AS s SELECT * FROM GRAPH_TABLE (g1 MATCH (src IS vl1 | vl2 | vl3)-[conn]->(dest) COLUMNS (src.vname AS svname, conn.ename AS cename, dest.vname AS dvname)); svname | cename | dvname --------+--------+-------- - v11 | e121 | v22 v12 | e122 | v21 + v11 | e121 | v22 v11 | e131 | v33 v11 | e132 | v31 v22 | e231 | v32 @@ -535,8 +535,8 @@ SELECT vn FROM all_vertices EXCEPT (SELECT svn FROM all_connected_vertices UNION SELECT sn, cn, dn FROM GRAPH_TABLE (g1 MATCH (src IS l1)-[conn IS l1]->(dest IS l1) COLUMNS (src.elname AS sn, conn.elname AS cn, dest.elname AS dn)); sn | cn | dn -----+------+----- - v11 | e121 | v22 v12 | e122 | v21 + v11 | e121 | v22 v11 | e131 | v33 v11 | e132 | v31 v22 | e231 | v32 diff --git a/src/test/regress/expected/join.out b/src/test/regress/expected/join.out index cfb50893585..692ffb2df38 100644 --- a/src/test/regress/expected/join.out +++ b/src/test/regress/expected/join.out @@ -2343,7 +2343,7 @@ where b.f1 = t.thousand and a.f1 = b.f1 and (a.f1+b.f1+999) = t.tenthous; ----------------------------------------------------------------------------------------------------------------- Nested Loop -> Nested Loop - Join Filter: ((sum(i4b.f1)) = ((sum(i4a.f1) + 1))) + Join Filter: (((sum(i4a.f1) + 1)) = (sum(i4b.f1))) -> Aggregate -> Seq Scan on int4_tbl i4a -> Aggregate @@ -8254,7 +8254,7 @@ ON sj_t1.id = _t2t3t4.id; Nested Loop Join Filter: (sj_t1.id = sj_t3.id) -> Nested Loop - Join Filter: (sj_t3.id = sj_t2_1.id) + Join Filter: (sj_t2_1.id = sj_t3.id) -> Nested Loop Join Filter: (sj_t2.id = sj_t3.id) -> Nested Loop -- 2.55.0