From 96a2e26d79ea03266b47ad89e7bc028094db5066 Mon Sep 17 00:00:00 2001 From: Matheus Alcantara Date: Mon, 13 Jul 2026 16:11:39 -0300 Subject: [PATCH v5 5/5] Cost bloom filter combinations for real , and stop capping candidates by selectivity find_interesting_bloom_filters() used to compute every candidate filter's selectivity and then discard all but the bloom_filter_pushdown_max most selective ones, so a filter that individually looked weak could never be pushed down even if it would have been valuable in combination with others, or valuable to a smart consumer for reasons the raw selectivity estimate can't see. It now returns every candidate that clears bloom_filter_pushdown_threshold, with no count-based trim, and generate_expected_filter_paths() grows combinations of them breadth-first by increasing size instead of enumerating every non-empty subset, stopping a combination's growth once its combined selectivity drops below the new bloom_filter_pushdown_combination_floor GUC. bloom_filter_pushdown_max is repurposed from "candidates kept" to "filters combined into one path variant", which together with the floor keeps the search bounded without having to guess up front which candidates matter. Base-relation scan paths are now built by their real constructors (create_seqscan_path, create_samplescan_path, create_bitmap_heap_path, create_tidscan_path, create_tidrangescan_path) instead of cloning with the filters passed straight in, and the new apply_expected_filters() helper charges a real per-tuple probe cost, one cpu_operator_cost per filter, on top of the row-count adjustment, so a combination only wins when it is actually cheaper rather than just because filtering was modeled as free. create_filtered_scan_path() is kept only for IndexPath, which still isn't worth rebuilding from scratch since create_index_path() would just re-derive indexclauses and pathkeys for no benefit (perhaps wen ca improve this). CustomPath is no longer touched by core for any of this. A provider that wants filter-aware CustomPaths now builds them itself, inside its own set_rel_pathlist_hook, by calling the newly exported find_bloom_filter_combinations() and costing the result however it likes, with real smart costing, or by calling apply_expected_filters() on its own freshly-built path if it doesn't want to be smart. test_bloom_customscan is updated to do exactly that, and gains a test with two independently-sourced filters to exercise a provider combining more than one filter into a single path. --- src/backend/optimizer/path/allpaths.c | 306 +++++++++++------- src/backend/optimizer/path/costsize.c | 19 +- src/backend/optimizer/path/indxpath.c | 4 +- src/backend/optimizer/path/tidpath.c | 8 +- src/backend/optimizer/plan/planner.c | 2 +- src/backend/optimizer/util/pathnode.c | 114 +++---- src/backend/utils/misc/guc_parameters.dat | 14 +- src/backend/utils/misc/postgresql.conf.sample | 5 +- src/include/optimizer/cost.h | 1 + src/include/optimizer/pathnode.h | 15 +- src/include/optimizer/paths.h | 1 + .../expected/test_bloom_customscan.out | 92 +++++- .../sql/test_bloom_customscan.sql | 51 ++- .../test_bloom_customscan.c | 74 +++-- 14 files changed, 491 insertions(+), 215 deletions(-) diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/path/allpaths.c index efc5c3b64d6..46b8162cc4d 100644 --- a/src/backend/optimizer/path/allpaths.c +++ b/src/backend/optimizer/path/allpaths.c @@ -872,7 +872,7 @@ set_plain_rel_pathlist(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte) return; /* Consider sequential scan */ - add_path(rel, create_seqscan_path(root, rel, required_outer, 0)); + add_path(rel, create_seqscan_path(root, rel, required_outer, 0, NIL)); /* If appropriate, consider parallel sequential scan */ if (rel->consider_parallel && required_outer == NULL) @@ -899,7 +899,7 @@ create_plain_partial_paths(PlannerInfo *root, RelOptInfo *rel) return; /* Add an unordered partial path based on a parallel sequential scan. */ - add_partial_path(rel, create_seqscan_path(root, rel, NULL, parallel_workers)); + add_partial_path(rel, create_seqscan_path(root, rel, NULL, parallel_workers, NIL)); } /* @@ -1019,9 +1019,8 @@ bloom_filter_recipient_reachable(PlannerInfo *root, Index owner_relid, * fraction is estimated as the semijoin selectivity of those clauses. * * A candidate is "interesting" only if it is expected to eliminate at least - * bloom_filter_pushdown_threshold of the rel's tuples. We keep at most - * bloom_filter_pushdown_max of the most selective candidates, and return them - * as a list of ExpectedFilter nodes. + * bloom_filter_pushdown_threshold of the rel's tuples. Every candidate that + * clears that bar is returned as an ExpectedFilter node. * * XXX This needs to be careful to not interfere with the general selectivity * estimation, performed by clauselist_selectivity(). We'll estimate the filter @@ -1200,92 +1199,66 @@ find_interesting_bloom_filters(PlannerInfo *root, RelOptInfo *rel) } } - /* - * We only connsider a limited number of interesting filters, to prevent - * path explosion. If we found too many, keep only the most selective ones - * (with smallest surviving fraction of tuples), to bound the number of - * generated paths. - * - * XXX This also aligns with good join orders - those tend to perform the - * most selective joins first. So we get to build the filters soon, even - * if the hashjoin optimization is not disabled. - */ - while (list_length(result) > bloom_filter_pushdown_max) - { - ExpectedFilter *worst = NULL; - ListCell *lcw; - - foreach(lcw, result) - { - ExpectedFilter *f = (ExpectedFilter *) lfirst(lcw); - - if (worst == NULL || f->selectivity > worst->selectivity) - worst = f; - } - result = list_delete_ptr(result, worst); - } - return result; } +/* + * A combination of interesting Bloom filters being grown by + * find_bloom_filter_combinations(), one candidate at a time. + * + * last_index is the position (in the candidate array) of the most recently + * added filter. Combinations are only ever extended with a candidate whose + * index is greater than last_index, which guarantees each combination is + * reached via exactly one path through the size-by-size generation below, + * so no combination is produced (or considered for pruning) more than once. + */ +typedef struct BloomFilterCombination +{ + List *filters; + int last_index; +} BloomFilterCombination; + /* * generate_expected_filter_paths * Generate additional scan paths that anticipate one or more pushed-down * Bloom filters. * - * For each non-empty subset of the interesting filters, we clone every eligible - * existing scan path, reducing its row estimate by the combined selectivity and - * attaching the corresponding ExpectedFilter nodes. + * For each combination of the interesting filters that we're willing to try + * (see find_bloom_filter_combinations), we build a genuinely new path via the + * real constructor for that path's type, passing the filters straight in so + * the constructor's own apply_expected_filters() call charges a real per-tuple + * probe cost, not just a row-count adjustment on a byte-copy. + * + * XXX: IndexPath is the one exception: it still goes through + * create_filtered_scan_path's clone-and-derate approach, since + * create_index_path() would re-derive indexclauses/pathkeys from scratch for + * no benefit (see the comment on create_filtered_scan_path). * * These paths are kept alongside the regular paths (add_path keeps paths with * differing expected_filters) and are consumed by join path generation; * set_cheapest never selects them. * - * XXX We must not clone paths that already have expected filters. - * - * XXX The cloning is a rather dirty way to copy paths. It does not readjust the - * cost in a reasonable way. For example custom scans could do something smart - * with the filters, so it should have a chance to deal with that. A cleaner - * solution might be to actually pass the filters to the various "create" - * function, like create_seqscan_path/... For CustomScan nodes we can probably - * do most of this in the set_rel_pathlist_hook, somewhere. Maybe that needs - * some helper methods, though. And maybe it will need to pass some of the info - * through the callbacks? Not sure, someone has to try that. - * - * XXX This may need some major changes to work with custom scans. Right now we - * only consider filters exactly matching the hash keys, so if the hashjoin is - * on (t1.a = t2.a AND t1.b = t2.b), then the filter will be on (a,b). But a - * custom scan may prefer "split" filters on each column independently. We'd - * need a way for the custom scan to indicate that, and we'd need to apply this - * only to the "matching" scan paths (and not to any other scan paths). But - * we only look at the paths after selecting the "interesting" filters, so we'd - * need to rethink that - we'd need to make the "interesting" filters specific - * to a path, or something like that. + * CustomPath is not handled here at all. Core cannot safely copy or construct + * a CustomPath, so a provider that wants filter-aware CustomPaths must build + * them itself, inside its own set_rel_pathlist_hook, by calling + * find_bloom_filter_combinations() directly (exported for that reason) and + * costing the result however it likes -- with real smart costing, or by + * calling apply_expected_filters() on its own freshly-built CustomPath if it + * doesn't want to be smart. */ static void generate_expected_filter_paths(PlannerInfo *root, RelOptInfo *rel) { - List *filters; + List *combinations; List *basepaths = NIL; - int nfilters; - uint32 combo; - ListCell *lc; - - filters = find_interesting_bloom_filters(root, rel); - if (filters == NIL) - return; - - nfilters = list_length(filters); /* * Snapshot the existing unparameterized, non-partial scan paths of a * supported type. We must snapshot before calling add_path(), which * mutates rel->pathlist. */ - foreach(lc, rel->pathlist) + foreach_ptr(Path, path, rel->pathlist) { - Path *path = (Path *) lfirst(lc); - /* XXX Is parameterization really a problem? Always? */ if (path->param_info != NULL || path->expected_filters != NIL) continue; @@ -1299,19 +1272,6 @@ generate_expected_filter_paths(PlannerInfo *root, RelOptInfo *rel) case T_TidRangePath: basepaths = lappend(basepaths, path); break; - case T_CustomPath: - - /* - * A base-relation CustomScan can receive a pushed-down filter, - * but only if the provider advertised that it knows how to - * consume one (it probes the filter itself inside its scan - * loop; see find_bloom_filter_recipient in createplan.c and - * ExecInitCustomScan). Providers that don't opt in, like heap, - * are unaffected. - */ - if (((CustomPath *) path)->flags & CUSTOMPATH_SUPPORT_BLOOM_FILTERS) - basepaths = lappend(basepaths, path); - break; default: break; } @@ -1320,51 +1280,169 @@ generate_expected_filter_paths(PlannerInfo *root, RelOptInfo *rel) if (basepaths == NIL) return; + combinations = find_bloom_filter_combinations(root, rel); + if (combinations == NIL) + return; + /* - * Generate all combinations of the interesting filters. We do that by - * iterating 1 to (2^n-1), which generates all bitmask in between. Those - * are the subsets. - * - * XXX This is a good demonstration why we need to keep the number of - * filters low - * - * XXX Maybe we should also stop adding filters once the other filters - * already eliminate enought tuples. Say, we know F1 alone eliminates 99% - * tuples. Does it make sense to also consider [F1,F2]? Probably not. We - * could track "maximum" sets, and reject combinations containing one - * of those. We'd need to generate sets of increasing size, the iteration - * does not do that. But that's not hard. + * For each combination, build a fresh path from each eligible base path + * via that base path's real constructor (except IndexPath, see the + * function comment above). */ - for (combo = 1; combo < ((uint32) 1 << nfilters); combo++) + foreach_ptr(List, combo, combinations) { - List *subset = NIL; - int i = 0; - ListCell *lcf; - - foreach(lcf, filters) + foreach_ptr(Path, base, basepaths) { - if (combo & ((uint32) 1 << i)) - subset = lappend(subset, lfirst(lcf)); - i++; - } + Path *newpath = NULL; - /* - * All filtered paths for this combo share the same expected_filters - * list. That's safe: the list is never modified, and add_path() only - * ever frees the Path node itself, not its expected_filters. - */ - foreach(lc, basepaths) - { - Path *base = (Path *) lfirst(lc); - Path *newpath; + switch (nodeTag(base)) + { + case T_Path: + if (base->pathtype == T_SeqScan) + newpath = create_seqscan_path(root, rel, NULL, + base->parallel_workers, + combo); + else if (base->pathtype == T_SampleScan) + newpath = create_samplescan_path(root, rel, NULL, + combo); + break; + case T_IndexPath: + // XXX: should not use copy? + newpath = create_filtered_scan_path(root, base, combo); + break; + case T_BitmapHeapPath: + newpath = (Path *) + create_bitmap_heap_path(root, rel, + ((BitmapHeapPath *) base)->bitmapqual, + NULL, 1.0, + base->parallel_workers, + combo); + break; + case T_TidPath: + newpath = (Path *) + create_tidscan_path(root, rel, + ((TidPath *) base)->tidquals, + NULL, combo); + break; + case T_TidRangePath: + newpath = (Path *) + create_tidrangescan_path(root, rel, + ((TidRangePath *) base)->tidrangequals, + NULL, base->parallel_workers, + combo); + break; + default: + break; + } - newpath = create_filtered_scan_path(root, base, subset); if (newpath != NULL) add_path(rel, newpath); } } } +/* + * find_bloom_filter_combinations + * Find the interesting Bloom filters rel could receive from a hash join + * above it (see find_interesting_bloom_filters), and return every + * combination of them worth building a path for, as a list of lists of + * ExpectedFilter. + * + * Combinations are generated breadth-first by increasing size (1, 2, 3, + * ...) rather than as every non-empty subset: a combination is only + * extended to the next size while (a) it has not yet reached max_size + * filters, and (b) its combined selectivity is still at or above + * selectivity_floor. Once a combination's expected surviving fraction + * drops below that floor, adding more filters to it has little further + * benefit, so it stops being grown (though its own combination, at its + * current size, is still included in the result). This bounds the number + * of combinations without having to pre-select which candidates are "best". + * + * Exported so a CustomScan provider can call it directly from its own + * set_rel_pathlist_hook (see generate_expected_filter_paths below); a + * caller that wants different grouping/pruning logic can bypass this and + * write its own on top of find_interesting_bloom_filters(), but that + * function itself stays static since every current caller wants exactly + * this combination logic. + */ +List * +find_bloom_filter_combinations(PlannerInfo *root, RelOptInfo *rel) +{ + List *result = NIL; + List *frontier = NIL; + int i; + int size; + List *filters = find_interesting_bloom_filters(root, rel); + int nfilters = list_length(filters); + + if (filters == NIL) + return NIL; + + /* + * Size 1: one combination per candidate filter. Every one of these is + * included in the result regardless of the floor check below -- the floor + * only decides whether we try to grow it into a size-2..max_size + * combination, not whether it belongs in the result. + */ + for (i = 0; i < nfilters; i++) + { + BloomFilterCombination *c = palloc(sizeof(BloomFilterCombination)); + + c->filters = list_make1((ExpectedFilter*) list_nth(filters, i)); + c->last_index = i; + + result = lappend(result, c->filters); + + if (expected_filters_selectivity(c->filters) >= bloom_filter_pushdown_combination_floor) + frontier = lappend(frontier, c); + } + + /* + * Sizes 2..max_size: extend every combination that survived the floor + * check at the previous size, one candidate at a time. The loop stops + * early if frontier ever becomes empty (every combination at the + * current size fell below the floor), independently of max_size. + * + * XXX: This may need optimizations, too much List's allocation? + */ + for (size = 2; size <= bloom_filter_pushdown_max && frontier != NIL; size++) + { + List *next_frontier = NIL; + + foreach_ptr(BloomFilterCombination, parent, frontier) + { + /* + * Only extend with candidates whose index sorts after every + * filter already in "parent" (i.e. after parent->last_index) -- + * see the struct comment above. + */ + for (i = parent->last_index + 1; i < nfilters; i++) + { + BloomFilterCombination *child = palloc(sizeof(BloomFilterCombination)); + + /* + * "parent" may be extended again by the next loop iteration + * (with a different filter), so its filter list must not be + * mutated in place, so list_copy() gives this child its own + * list to extend. + */ + child->filters = lappend(list_copy(parent->filters), list_nth(filters, i)); + child->last_index = i; + + result = lappend(result, child->filters); + + if (expected_filters_selectivity(child->filters) >= bloom_filter_pushdown_combination_floor) + next_frontier = lappend(next_frontier, child); + } + } + + frontier = next_frontier; + } + + return result; +} + + /* * set_tablesample_rel_size * Set size estimates for a sampled relation @@ -1423,7 +1501,7 @@ set_tablesample_rel_pathlist(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry * required_outer = rel->lateral_relids; /* Consider sampled scan */ - path = create_samplescan_path(root, rel, required_outer); + path = create_samplescan_path(root, rel, required_outer, NIL); /* * If the sampling method does not support repeatable scans, we must avoid @@ -5428,7 +5506,7 @@ create_partial_bitmap_paths(PlannerInfo *root, RelOptInfo *rel, return; add_partial_path(rel, (Path *) create_bitmap_heap_path(root, rel, - bitmapqual, rel->lateral_relids, 1.0, parallel_workers)); + bitmapqual, rel->lateral_relids, 1.0, parallel_workers, NIL)); } /* diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c index a29e63e8738..27292f0d51d 100644 --- a/src/backend/optimizer/path/costsize.c +++ b/src/backend/optimizer/path/costsize.c @@ -176,14 +176,23 @@ bool enable_async_append = true; double bloom_filter_pushdown_threshold = 0.3; /* - * Upper bound on the number of distinct interesting Bloom filters considered - * for a single scan relation. This bounds the number of additional paths - * generated per scan (the planner enumerates non-empty subsets of the - * interesting filters, i.e. up to 2^bloom_filter_pushdown_max - 1 extra - * paths per base scan path). + * Upper bound on the number of interesting Bloom filters that may be + * combined into a single filter-aware scan path variant for a base + * relation. This does not limit how many candidate filters are considered + * (see bloom_filter_pushdown_threshold) -- only how large a combination of + * them the planner is willing to build a path for. */ int bloom_filter_pushdown_max = 3; +/* + * Once the combined selectivity of a growing set of Bloom filters falls below + * this fraction of surviving rows, the planner stops trying to add further + * filters to that combination. The marginal benefit of additional filtering is + * likely small, so it's not worth the extra path and extra per-tuple filter + * evaluation cost. + */ +double bloom_filter_pushdown_combination_floor = 0.1; + typedef struct { PlannerInfo *root; diff --git a/src/backend/optimizer/path/indxpath.c b/src/backend/optimizer/path/indxpath.c index 3f5d4fa3182..a6c9a41ead1 100644 --- a/src/backend/optimizer/path/indxpath.c +++ b/src/backend/optimizer/path/indxpath.c @@ -344,7 +344,7 @@ create_index_paths(PlannerInfo *root, RelOptInfo *rel) bitmapqual = choose_bitmap_and(root, rel, bitindexpaths); bpath = create_bitmap_heap_path(root, rel, bitmapqual, - rel->lateral_relids, 1.0, 0); + rel->lateral_relids, 1.0, 0, NIL); add_path(rel, (Path *) bpath); /* create a partial bitmap heap path */ @@ -411,7 +411,7 @@ create_index_paths(PlannerInfo *root, RelOptInfo *rel) required_outer = PATH_REQ_OUTER(bitmapqual); loop_count = get_loop_count(root, rel->relid, required_outer); bpath = create_bitmap_heap_path(root, rel, bitmapqual, - required_outer, loop_count, 0); + required_outer, loop_count, 0, NIL); add_path(rel, (Path *) bpath); } } diff --git a/src/backend/optimizer/path/tidpath.c b/src/backend/optimizer/path/tidpath.c index 18a3654720b..83b261542ea 100644 --- a/src/backend/optimizer/path/tidpath.c +++ b/src/backend/optimizer/path/tidpath.c @@ -468,7 +468,7 @@ BuildParameterizedTidPaths(PlannerInfo *root, RelOptInfo *rel, List *clauses) required_outer = bms_del_member(required_outer, rel->relid); add_path(rel, (Path *) create_tidscan_path(root, rel, tidquals, - required_outer)); + required_outer, NIL)); } } @@ -520,7 +520,7 @@ create_tidscan_paths(PlannerInfo *root, RelOptInfo *rel) Relids required_outer = rel->lateral_relids; add_path(rel, (Path *) create_tidscan_path(root, rel, tidquals, - required_outer)); + required_outer, NIL)); /* * When the qual is CurrentOfExpr, the path that we just added is the @@ -554,7 +554,7 @@ create_tidscan_paths(PlannerInfo *root, RelOptInfo *rel) add_path(rel, (Path *) create_tidrangescan_path(root, rel, tidrangequals, required_outer, - 0)); + 0, NIL)); /* If appropriate, consider parallel tid range scan. */ if (rel->consider_parallel && required_outer == NULL) @@ -569,7 +569,7 @@ create_tidscan_paths(PlannerInfo *root, RelOptInfo *rel) rel, tidrangequals, required_outer, - parallel_workers)); + parallel_workers, NIL)); } } diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c index 8bb0948c4b8..956067cb187 100644 --- a/src/backend/optimizer/plan/planner.c +++ b/src/backend/optimizer/plan/planner.c @@ -7177,7 +7177,7 @@ plan_cluster_use_sort(Oid tableOid, Oid indexOid) comparisonCost = 2.0 * (indexExprCost.startup + indexExprCost.per_tuple); /* Estimate the cost of seq scan + sort */ - seqScanPath = create_seqscan_path(root, rel, NULL, 0); + seqScanPath = create_seqscan_path(root, rel, NULL, 0, NIL); cost_sort(&seqScanAndSortPath, root, NIL, seqScanPath->disabled_nodes, seqScanPath->total_cost, rel->tuples, rel->reltarget->width, diff --git a/src/backend/optimizer/util/pathnode.c b/src/backend/optimizer/util/pathnode.c index 8c2ecdbcb8f..93a68e1a554 100644 --- a/src/backend/optimizer/util/pathnode.c +++ b/src/backend/optimizer/util/pathnode.c @@ -459,10 +459,33 @@ expected_filters_selectivity(List *filters) return sel; } +/* + * apply_expected_filters + * Adjust an already-costed path to account for probing the given set of + * Bloom filters against every tuple it produces, and record them as the + * path's expected_filters. + * + * This is the generic, not-smart costing fallback: it assumes each filter + * costs one hash-probe-equivalent (cpu_operator_cost) per input tuple, on + * top of shrinking the row estimate by the filters' combined selectivity. + * Callers that can do something smarter (e.g. a CustomScan skipping whole + * row groups via a zone map) should cost themselves instead of using this. + */ +void +apply_expected_filters(Path *path, List *filters) +{ + if (filters == NIL) + return; + + path->total_cost += list_length(filters) * cpu_operator_cost * path->rows; + path->rows = clamp_row_est(path->rows * expected_filters_selectivity(filters)); + path->expected_filters = filters; +} + /* * create_filtered_scan_path - * Build a copy of a base-relation scan path that additionally expects the - * given set of pushed-down Bloom filters. + * Build a copy of an IndexPath that additionally expects the given set of + * pushed-down Bloom filters. * * The clone shares all substructure with the original path (parent, * pathtarget, clauses, etc.); only the rows estimate is reduced to reflect @@ -470,61 +493,30 @@ expected_filters_selectivity(List *filters) * safe because create_plan() treats the clone identically to the original * (it ignores expected_filters), and add_path() may freely pfree the clone. * - * Only the plain scan path node types that can receive a pushed-down filter - * are supported (matching find_bloom_filter_recipient in createplan.c). - * Returns NULL for unsupported path types. - * - * XXX This should probably adjust the CPU cost in some way. It assumes the - * filter checks are free, which does not seem right. + * Every other base-relation scan path type is instead built directly by its + * real constructor with the filters passed in (see create_seqscan_path, + * create_bitmap_heap_path, etc., and apply_expected_filters above); IndexPath + * is the one exception, kept on this cloning path for now because + * create_index_path() would re-derive indexclauses/pathkeys from scratch -- + * see the comment on the T_IndexScan/T_IndexOnlyScan case in + * reparameterize_path() for the same tradeoff made there. */ Path * create_filtered_scan_path(PlannerInfo *root, Path *subpath, List *filters) { - Path *newpath; - size_t sz; + IndexPath *newpath; - switch (nodeTag(subpath)) - { - case T_Path: - /* plain seqscan/samplescan etc. */ - sz = sizeof(Path); - break; - case T_IndexPath: - sz = sizeof(IndexPath); - break; - case T_BitmapHeapPath: - sz = sizeof(BitmapHeapPath); - break; - case T_TidPath: - sz = sizeof(TidPath); - break; - case T_TidRangePath: - sz = sizeof(TidRangePath); - break; - case T_CustomPath: - - /* - * A base-relation CustomScan provider that advertised - * CUSTOMPATH_SUPPORT_BLOOM_FILTERS can receive a pushed-down - * filter and apply it in its own scan loop - * .generate_expected_filter_paths() only offers such paths here, - * so we need not re-check the flag. - */ - sz = sizeof(CustomPath); - break; - default: - /* unsupported scan path type */ - return NULL; - } + if (!IsA(subpath, IndexPath)) + return NULL; - newpath = (Path *) palloc(sz); - memcpy(newpath, subpath, sz); + newpath = (IndexPath *) palloc(sizeof(IndexPath)); + memcpy(newpath, subpath, sizeof(IndexPath)); - newpath->expected_filters = filters; - newpath->rows = clamp_row_est(subpath->rows * - expected_filters_selectivity(filters)); + newpath->path.expected_filters = filters; + newpath->path.rows = clamp_row_est(subpath->rows * + expected_filters_selectivity(filters)); - return newpath; + return (Path *) newpath; } /* @@ -1193,7 +1185,8 @@ add_partial_path_precheck(RelOptInfo *parent_rel, int disabled_nodes, */ Path * create_seqscan_path(PlannerInfo *root, RelOptInfo *rel, - Relids required_outer, int parallel_workers) + Relids required_outer, int parallel_workers, + List *filters) { Path *pathnode = makeNode(Path); @@ -1208,6 +1201,7 @@ create_seqscan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->pathkeys = NIL; /* seqscan has unordered result */ cost_seqscan(pathnode, root, rel, pathnode->param_info); + apply_expected_filters(pathnode, filters); return pathnode; } @@ -1217,7 +1211,8 @@ create_seqscan_path(PlannerInfo *root, RelOptInfo *rel, * Creates a path node for a sampled table scan. */ Path * -create_samplescan_path(PlannerInfo *root, RelOptInfo *rel, Relids required_outer) +create_samplescan_path(PlannerInfo *root, RelOptInfo *rel, + Relids required_outer, List *filters) { Path *pathnode = makeNode(Path); @@ -1232,6 +1227,7 @@ create_samplescan_path(PlannerInfo *root, RelOptInfo *rel, Relids required_outer pathnode->pathkeys = NIL; /* samplescan has unordered result */ cost_samplescan(pathnode, root, rel, pathnode->param_info); + apply_expected_filters(pathnode, filters); return pathnode; } @@ -1320,7 +1316,8 @@ create_bitmap_heap_path(PlannerInfo *root, Path *bitmapqual, Relids required_outer, double loop_count, - int parallel_degree) + int parallel_degree, + List *filters) { BitmapHeapPath *pathnode = makeNode(BitmapHeapPath); @@ -1339,6 +1336,7 @@ create_bitmap_heap_path(PlannerInfo *root, cost_bitmap_heap_scan(&pathnode->path, root, rel, pathnode->path.param_info, bitmapqual, loop_count); + apply_expected_filters(&pathnode->path, filters); return pathnode; } @@ -1453,7 +1451,7 @@ create_bitmap_or_path(PlannerInfo *root, */ TidPath * create_tidscan_path(PlannerInfo *root, RelOptInfo *rel, List *tidquals, - Relids required_outer) + Relids required_outer, List *filters) { TidPath *pathnode = makeNode(TidPath); @@ -1471,6 +1469,7 @@ create_tidscan_path(PlannerInfo *root, RelOptInfo *rel, List *tidquals, cost_tidscan(&pathnode->path, root, rel, tidquals, pathnode->path.param_info); + apply_expected_filters(&pathnode->path, filters); return pathnode; } @@ -1483,7 +1482,7 @@ create_tidscan_path(PlannerInfo *root, RelOptInfo *rel, List *tidquals, TidRangePath * create_tidrangescan_path(PlannerInfo *root, RelOptInfo *rel, List *tidrangequals, Relids required_outer, - int parallel_workers) + int parallel_workers, List *filters) { TidRangePath *pathnode = makeNode(TidRangePath); @@ -1501,6 +1500,7 @@ create_tidrangescan_path(PlannerInfo *root, RelOptInfo *rel, cost_tidrangescan(&pathnode->path, root, rel, tidrangequals, pathnode->path.param_info); + apply_expected_filters(&pathnode->path, filters); return pathnode; } @@ -4115,9 +4115,9 @@ reparameterize_path(PlannerInfo *root, Path *path, switch (path->pathtype) { case T_SeqScan: - return create_seqscan_path(root, rel, required_outer, 0); + return create_seqscan_path(root, rel, required_outer, 0, NIL); case T_SampleScan: - return create_samplescan_path(root, rel, required_outer); + return create_samplescan_path(root, rel, required_outer, NIL); case T_IndexScan: case T_IndexOnlyScan: { @@ -4145,7 +4145,7 @@ reparameterize_path(PlannerInfo *root, Path *path, rel, bpath->bitmapqual, required_outer, - loop_count, 0); + loop_count, 0, NIL); } case T_SubqueryScan: { diff --git a/src/backend/utils/misc/guc_parameters.dat b/src/backend/utils/misc/guc_parameters.dat index aacadc8de40..0950c1ae546 100644 --- a/src/backend/utils/misc/guc_parameters.dat +++ b/src/backend/utils/misc/guc_parameters.dat @@ -380,9 +380,19 @@ max => 'BLCKSZ', }, +{ name => 'bloom_filter_pushdown_combination_floor', type => 'real', context => 'PGC_USERSET', group => 'QUERY_TUNING_OTHER', + short_desc => 'Minimum surviving-row fraction below which additional bloom filters stop being combined.', + long_desc => 'Once the combined selectivity of a set of pushed-down bloom filters falls below this fraction, the planner stops trying to add more filters to that combination, since the marginal benefit of further filtering is likely to be small.', + flags => 'GUC_EXPLAIN', + variable => 'bloom_filter_pushdown_combination_floor', + boot_val => '0.1', + min => '0.0', + max => '1.0', +}, + { name => 'bloom_filter_pushdown_max', type => 'int', context => 'PGC_USERSET', group => 'QUERY_TUNING_OTHER', - short_desc => 'Maximum number of pushed-down hash join bloom filters considered per scan.', - long_desc => 'Bounds how many interesting bloom filters the planner enumerates subsets of when building filter-aware scan paths.', + short_desc => 'Maximum number of hash join bloom filters combined into a single pushed-down filter set.', + long_desc => 'Bounds how many interesting bloom filters the planner will combine together when building filter-aware scan path variants for a base relation.', flags => 'GUC_EXPLAIN', variable => 'bloom_filter_pushdown_max', boot_val => '3', diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 3247ed46e6e..195c7bbd77e 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -486,8 +486,9 @@ # - Other Planner Options - -#bloom_filter_pushdown_max = 3 # range 0-10 -#bloom_filter_pushdown_threshold = 0.3 # range 0.0-1.0 +#bloom_filter_pushdown_max = 3 # range 0-10 +#bloom_filter_pushdown_threshold = 0.3 # range 0.0-1.0 +#bloom_filter_pushdown_combination_floor = 0.1 # range 0.0-1.0 #default_statistics_target = 100 # range 1-10000 #constraint_exclusion = partition # on, off, or partition #cursor_tuple_fraction = 0.1 # range 0.0-1.0 diff --git a/src/include/optimizer/cost.h b/src/include/optimizer/cost.h index 2047f211c4c..7a1bbd577ac 100644 --- a/src/include/optimizer/cost.h +++ b/src/include/optimizer/cost.h @@ -74,6 +74,7 @@ extern PGDLLIMPORT bool enable_presorted_aggregate; extern PGDLLIMPORT bool enable_async_append; extern PGDLLIMPORT double bloom_filter_pushdown_threshold; extern PGDLLIMPORT int bloom_filter_pushdown_max; +extern PGDLLIMPORT double bloom_filter_pushdown_combination_floor; extern PGDLLIMPORT int constraint_exclusion; extern double index_pages_fetched(double tuples_fetched, BlockNumber pages, diff --git a/src/include/optimizer/pathnode.h b/src/include/optimizer/pathnode.h index 0dcae2ae3b3..7d7e080c0db 100644 --- a/src/include/optimizer/pathnode.h +++ b/src/include/optimizer/pathnode.h @@ -66,13 +66,15 @@ extern bool add_partial_path_precheck(RelOptInfo *parent_rel, extern bool expected_filters_equal(List *a, List *b); extern double expected_filters_selectivity(List *filters); +extern void apply_expected_filters(Path *path, List *filters); extern Path *create_filtered_scan_path(PlannerInfo *root, Path *subpath, List *filters); extern Path *create_seqscan_path(PlannerInfo *root, RelOptInfo *rel, - Relids required_outer, int parallel_workers); + Relids required_outer, int parallel_workers, + List *filters); extern Path *create_samplescan_path(PlannerInfo *root, RelOptInfo *rel, - Relids required_outer); + Relids required_outer, List *filters); extern IndexPath *create_index_path(PlannerInfo *root, IndexOptInfo *index, List *indexclauses, @@ -89,7 +91,8 @@ extern BitmapHeapPath *create_bitmap_heap_path(PlannerInfo *root, Path *bitmapqual, Relids required_outer, double loop_count, - int parallel_degree); + int parallel_degree, + List *filters); extern BitmapAndPath *create_bitmap_and_path(PlannerInfo *root, RelOptInfo *rel, List *bitmapquals); @@ -97,12 +100,14 @@ extern BitmapOrPath *create_bitmap_or_path(PlannerInfo *root, RelOptInfo *rel, List *bitmapquals); extern TidPath *create_tidscan_path(PlannerInfo *root, RelOptInfo *rel, - List *tidquals, Relids required_outer); + List *tidquals, Relids required_outer, + List *filters); extern TidRangePath *create_tidrangescan_path(PlannerInfo *root, RelOptInfo *rel, List *tidrangequals, Relids required_outer, - int parallel_workers); + int parallel_workers, + List *filters); extern AppendPath *create_append_path(PlannerInfo *root, RelOptInfo *rel, AppendPathInput input, diff --git a/src/include/optimizer/paths.h b/src/include/optimizer/paths.h index 636761f2e94..cc0fa6a010b 100644 --- a/src/include/optimizer/paths.h +++ b/src/include/optimizer/paths.h @@ -61,6 +61,7 @@ extern PGDLLIMPORT join_search_hook_type join_search_hook; extern RelOptInfo *make_one_rel(PlannerInfo *root, List *joinlist); extern RelOptInfo *standard_join_search(PlannerInfo *root, int levels_needed, List *initial_rels); +extern List *find_bloom_filter_combinations(PlannerInfo *root, RelOptInfo *rel); extern void generate_gather_paths(PlannerInfo *root, RelOptInfo *rel, bool override_rows); diff --git a/src/test/modules/test_bloom_customscan/expected/test_bloom_customscan.out b/src/test/modules/test_bloom_customscan/expected/test_bloom_customscan.out index 8e784209901..11eea4b134e 100644 --- a/src/test/modules/test_bloom_customscan/expected/test_bloom_customscan.out +++ b/src/test/modules/test_bloom_customscan/expected/test_bloom_customscan.out @@ -107,8 +107,98 @@ SELECT test_bloom_cs_rejected_rows() > 0 AS filter_rejected_rows; t (1 row) +-- Two independent joins: the provider should offer, and the planner should +-- choose, a CustomPath expecting a combined filter from both joins. +-- cpu_operator_cost is lowered so combining is robustly cheaper rather than a +-- near-tie. +CREATE TABLE cs_wide_a (id int, label text); +CREATE TABLE cs_wide_b (id int, label text); +INSERT INTO cs_wide_a SELECT g, 'a' || g FROM generate_series(1, 400) g; +INSERT INTO cs_wide_b SELECT g, 'b' || g FROM generate_series(1, 400) g; +ANALYZE cs_wide_a; +ANALYZE cs_wide_b; +SET test_bloom_customscan.enable = on; +SET enable_seqscan = off; +SET cpu_operator_cost = 0.0001; +EXPLAIN (COSTS OFF, VERBOSE) +SELECT count(*) FROM cs_fact f + JOIN cs_wide_a wa ON f.a = wa.id + JOIN cs_wide_b wb ON f.b = wb.id; + QUERY PLAN +---------------------------------------------------------------------------------- + Aggregate + Output: count(*) + -> Hash Join + Hash Cond: (f.b = wb.id) + -> Hash Join + Output: f.b + Hash Cond: (f.a = wa.id) + -> Custom Scan (TestBloomCustomScan) on public.cs_fact f + Output: f.a, f.b + -> Hash + Output: wa.id + Bloom Filter 1 + -> Custom Scan (TestBloomCustomScan) on public.cs_wide_a wa + Output: wa.id + -> Hash + Output: wb.id + Bloom Filter 2 + -> Custom Scan (TestBloomCustomScan) on public.cs_wide_b wb + Output: wb.id +(19 rows) + +SELECT test_bloom_cs_reset(); + test_bloom_cs_reset +--------------------- + +(1 row) + +SELECT count(*) FROM cs_fact f + JOIN cs_wide_a wa ON f.a = wa.id + JOIN cs_wide_b wb ON f.b = wb.id; + count +------- + 8000 +(1 row) + +SELECT test_bloom_cs_rejected_rows() > 0 AS filter_rejected_rows; + filter_rejected_rows +---------------------- + t +(1 row) + +-- Correctness: the result must be identical with and without the filters. +SET test_bloom_customscan.enable = on; +SET enable_seqscan = off; +CREATE TEMP TABLE r_cs2 AS + SELECT f.a, count(*) AS n FROM cs_fact f + JOIN cs_wide_a wa ON f.a = wa.id + JOIN cs_wide_b wb ON f.b = wb.id + GROUP BY f.a; +SET test_bloom_customscan.enable = off; +SET enable_seqscan = on; +CREATE TEMP TABLE r_plain2 AS + SELECT f.a, count(*) AS n FROM cs_fact f + JOIN cs_wide_a wa ON f.a = wa.id + JOIN cs_wide_b wb ON f.b = wb.id + GROUP BY f.a; +SELECT count(*) AS cs_minus_plain + FROM (SELECT * FROM r_cs2 EXCEPT SELECT * FROM r_plain2) x; + cs_minus_plain +---------------- + 0 +(1 row) + +SELECT count(*) AS plain_minus_cs + FROM (SELECT * FROM r_plain2 EXCEPT SELECT * FROM r_cs2) x; + plain_minus_cs +---------------- + 0 +(1 row) + -- cleanup +RESET cpu_operator_cost; SET test_bloom_customscan.enable = off; SET enable_seqscan = on; -DROP TABLE cs_fact, cs_dim; +DROP TABLE cs_fact, cs_dim, cs_wide_a, cs_wide_b; DROP EXTENSION test_bloom_customscan; diff --git a/src/test/modules/test_bloom_customscan/sql/test_bloom_customscan.sql b/src/test/modules/test_bloom_customscan/sql/test_bloom_customscan.sql index daae84a531a..8da2e0789e6 100644 --- a/src/test/modules/test_bloom_customscan/sql/test_bloom_customscan.sql +++ b/src/test/modules/test_bloom_customscan/sql/test_bloom_customscan.sql @@ -59,8 +59,57 @@ SELECT count(*) FROM cs_fact f JOIN cs_dim d ON f.a = d.id AND f.b = d.id2; SELECT test_bloom_cs_perkey_built() AS perkey_filters_built; SELECT test_bloom_cs_rejected_rows() > 0 AS filter_rejected_rows; +-- Two independent joins: the provider should offer, and the planner should +-- choose, a CustomPath expecting a combined filter from both joins. +-- cpu_operator_cost is lowered so combining is robustly cheaper rather than a +-- near-tie. +CREATE TABLE cs_wide_a (id int, label text); +CREATE TABLE cs_wide_b (id int, label text); +INSERT INTO cs_wide_a SELECT g, 'a' || g FROM generate_series(1, 400) g; +INSERT INTO cs_wide_b SELECT g, 'b' || g FROM generate_series(1, 400) g; +ANALYZE cs_wide_a; +ANALYZE cs_wide_b; + +SET test_bloom_customscan.enable = on; +SET enable_seqscan = off; +SET cpu_operator_cost = 0.0001; + +EXPLAIN (COSTS OFF, VERBOSE) +SELECT count(*) FROM cs_fact f + JOIN cs_wide_a wa ON f.a = wa.id + JOIN cs_wide_b wb ON f.b = wb.id; + +SELECT test_bloom_cs_reset(); +SELECT count(*) FROM cs_fact f + JOIN cs_wide_a wa ON f.a = wa.id + JOIN cs_wide_b wb ON f.b = wb.id; +SELECT test_bloom_cs_rejected_rows() > 0 AS filter_rejected_rows; + +-- Correctness: the result must be identical with and without the filters. +SET test_bloom_customscan.enable = on; +SET enable_seqscan = off; +CREATE TEMP TABLE r_cs2 AS + SELECT f.a, count(*) AS n FROM cs_fact f + JOIN cs_wide_a wa ON f.a = wa.id + JOIN cs_wide_b wb ON f.b = wb.id + GROUP BY f.a; + +SET test_bloom_customscan.enable = off; +SET enable_seqscan = on; +CREATE TEMP TABLE r_plain2 AS + SELECT f.a, count(*) AS n FROM cs_fact f + JOIN cs_wide_a wa ON f.a = wa.id + JOIN cs_wide_b wb ON f.b = wb.id + GROUP BY f.a; + +SELECT count(*) AS cs_minus_plain + FROM (SELECT * FROM r_cs2 EXCEPT SELECT * FROM r_plain2) x; +SELECT count(*) AS plain_minus_cs + FROM (SELECT * FROM r_plain2 EXCEPT SELECT * FROM r_cs2) x; + -- cleanup +RESET cpu_operator_cost; SET test_bloom_customscan.enable = off; SET enable_seqscan = on; -DROP TABLE cs_fact, cs_dim; +DROP TABLE cs_fact, cs_dim, cs_wide_a, cs_wide_b; DROP EXTENSION test_bloom_customscan; diff --git a/src/test/modules/test_bloom_customscan/test_bloom_customscan.c b/src/test/modules/test_bloom_customscan/test_bloom_customscan.c index 99338c7d361..30f53ff45c7 100644 --- a/src/test/modules/test_bloom_customscan/test_bloom_customscan.c +++ b/src/test/modules/test_bloom_customscan/test_bloom_customscan.c @@ -64,6 +64,8 @@ typedef struct BloomCSScanState } BloomCSScanState; /* forward declarations */ +static void bloom_cs_add_path(RelOptInfo *rel, Cost startup_cost, + Cost total_cost, List *filters); static Plan *bloom_cs_plan_custom_path(PlannerInfo *root, RelOptInfo *rel, CustomPath *best_path, List *tlist, List *clauses, List *custom_plans); @@ -93,18 +95,57 @@ static const CustomExecMethods bloom_cs_exec_methods = { }; /* - * set_rel_pathlist_hook: offer a bloom-filter-capable CustomPath for plain + * bloom_cs_add_path + * Build and add one bloom-filter-capable CustomPath, optionally expecting + * the given set of pushed-down filters. We don't do anything smart with + * the filters ourselves, so apply_expected_filters() supplies the same + * generic per-tuple probe cost and row-count adjustment core uses for + * stock scan types. + */ +static void +bloom_cs_add_path(RelOptInfo *rel, Cost startup_cost, Cost total_cost, + List *filters) +{ + CustomPath *cpath = makeNode(CustomPath); + + cpath->path.pathtype = T_CustomScan; + cpath->path.parent = rel; + cpath->path.pathtarget = rel->reltarget; + cpath->path.param_info = NULL; + cpath->path.parallel_aware = false; + cpath->path.parallel_safe = false; + cpath->path.parallel_workers = 0; + cpath->path.rows = rel->rows; + cpath->path.startup_cost = startup_cost; + cpath->path.total_cost = total_cost; + cpath->path.pathkeys = NIL; + + cpath->flags = CUSTOMPATH_SUPPORT_BLOOM_FILTERS; + cpath->custom_paths = NIL; + cpath->custom_private = NIL; + cpath->methods = &bloom_cs_path_methods; + + apply_expected_filters(&cpath->path, filters); + + add_path(rel, (Path *) cpath); +} + +/* + * set_rel_pathlist_hook: offer bloom-filter-capable CustomPaths for plain * heap base relations. We copy the cost of the existing sequential scan path - * so join costing stays sane; the test forces the custom scan to be chosen - * with "SET enable_seqscan = off" (the CustomPath keeps disabled_nodes = 0). + * so join costing stays sane. + * + * Besides the plain, filter-oblivious path, we also offer one CustomPath per + * combination of interesting Bloom filters this rel could receive from a + * hash join above it. */ static void bloom_cs_set_rel_pathlist(PlannerInfo *root, RelOptInfo *rel, Index rti, RangeTblEntry *rte) { - CustomPath *cpath; Cost startup_cost = 0; Cost total_cost = 0; + List *combinations; ListCell *lc; if (prev_set_rel_pathlist_hook) @@ -132,25 +173,16 @@ bloom_cs_set_rel_pathlist(PlannerInfo *root, RelOptInfo *rel, Index rti, } } - cpath = makeNode(CustomPath); - cpath->path.pathtype = T_CustomScan; - cpath->path.parent = rel; - cpath->path.pathtarget = rel->reltarget; - cpath->path.param_info = NULL; - cpath->path.parallel_aware = false; - cpath->path.parallel_safe = false; - cpath->path.parallel_workers = 0; - cpath->path.rows = rel->rows; - cpath->path.startup_cost = startup_cost; - cpath->path.total_cost = total_cost; - cpath->path.pathkeys = NIL; + bloom_cs_add_path(rel, startup_cost, total_cost, NIL); - cpath->flags = CUSTOMPATH_SUPPORT_BLOOM_FILTERS; - cpath->custom_paths = NIL; - cpath->custom_private = NIL; - cpath->methods = &bloom_cs_path_methods; + combinations = find_bloom_filter_combinations(root, rel); - add_path(rel, (Path *) cpath); + foreach(lc, combinations) + { + List *combo = (List *) lfirst(lc); + + bloom_cs_add_path(rel, startup_cost, total_cost, combo); + } } static Plan * -- 2.50.1 (Apple Git-155)