From 9618e3b0bd0b04e96ac26af6b8e1b67b6ac52d07 Mon Sep 17 00:00:00 2001 From: Tomas Vondra Date: Sat, 1 Aug 2026 15:38:12 +0200 Subject: [PATCH v10 20/21] Proper construction of paths with expected filters * Cost filter-aware scan paths for real. Paths are built through the normal create_*_path constructors, and apply_expected_filters() charges one cpu_operator_cost per filter per tuple, so filtering is no longer modeled as free. A filter is chosen only when the resulting path is actually cheaper. * Add an IndexPath filter variant. create_filtered_scan_path() clones the source IndexPath, reusing its indexclauses and pathkeys, and applies the same probe-cost and row adjustment as the other scan types. Co-authored-by: Rui Zhao --- .../pg_plan_advice/expected/join_order.out | 20 +- .../expected/pg_stash_advice.out | 40 ++-- src/backend/optimizer/path/allpaths.c | 105 ++++++----- src/backend/optimizer/path/costsize.c | 122 +++++++++++- src/backend/optimizer/path/indxpath.c | 6 +- src/backend/optimizer/path/tidpath.c | 8 +- src/backend/optimizer/plan/planner.c | 2 +- src/backend/optimizer/util/pathnode.c | 178 +++++++++++------- src/include/optimizer/cost.h | 21 ++- src/include/optimizer/pathnode.h | 16 +- .../expected/test_bloom_customscan.out | 9 +- src/test/regress/expected/hashjoin_bloom.out | 4 +- .../expected/hashjoin_bloom_snowflake.out | 22 +-- .../regress/expected/hashjoin_bloom_star.out | 64 +++---- src/test/regress/expected/join.out | 58 +++--- src/test/regress/expected/misc_functions.out | 14 +- src/test/regress/expected/select_parallel.out | 37 ++-- 17 files changed, 444 insertions(+), 282 deletions(-) diff --git a/contrib/pg_plan_advice/expected/join_order.out b/contrib/pg_plan_advice/expected/join_order.out index ded25c4010b..f4c0fe60027 100644 --- a/contrib/pg_plan_advice/expected/join_order.out +++ b/contrib/pg_plan_advice/expected/join_order.out @@ -34,14 +34,12 @@ SELECT * FROM jo_fact f -> Hash Join Hash Cond: (f.dim1_id = d1.id) -> Seq Scan on jo_fact f - Bloom Filter 2: keys=(dim2_id) - Bloom Filter 1: keys=(dim1_id) + Bloom Filter 1: keys=(dim2_id) -> Hash - Bloom Filter 1 -> Seq Scan on jo_dim1 d1 Filter: (val1 = 1) -> Hash - Bloom Filter 2 + Bloom Filter 1 -> Seq Scan on jo_dim2 d2 Filter: (val2 = 1) Generated Plan Advice: @@ -49,7 +47,7 @@ SELECT * FROM jo_fact f HASH_JOIN(d1 d2) SEQ_SCAN(f d1 d2) NO_GATHER(f d1 d2) -(20 rows) +(18 rows) -- Force a few different join orders. Some of these are very inefficient, -- but the planner considers them all viable. @@ -67,14 +65,12 @@ SELECT * FROM jo_fact f -> Hash Join Hash Cond: (f.dim1_id = d1.id) -> Seq Scan on jo_fact f - Bloom Filter 2: keys=(dim2_id) - Bloom Filter 1: keys=(dim1_id) + Bloom Filter 1: keys=(dim2_id) -> Hash - Bloom Filter 1 -> Seq Scan on jo_dim1 d1 Filter: (val1 = 1) -> Hash - Bloom Filter 2 + Bloom Filter 1 -> Seq Scan on jo_dim2 d2 Filter: (val2 = 1) Supplied Plan Advice: @@ -84,7 +80,7 @@ SELECT * FROM jo_fact f HASH_JOIN(d1 d2) SEQ_SCAN(f d1 d2) NO_GATHER(f d1 d2) -(22 rows) +(20 rows) SET LOCAL pg_plan_advice.advice = 'join_order(f d2 d1)'; EXPLAIN (COSTS OFF, PLAN_ADVICE) @@ -100,13 +96,11 @@ SELECT * FROM jo_fact f Hash Cond: (f.dim2_id = d2.id) -> Seq Scan on jo_fact f Bloom Filter 1: keys=(dim2_id) - Bloom Filter 2: keys=(dim1_id) -> Hash Bloom Filter 1 -> Seq Scan on jo_dim2 d2 Filter: (val2 = 1) -> Hash - Bloom Filter 2 -> Seq Scan on jo_dim1 d1 Filter: (val1 = 1) Supplied Plan Advice: @@ -116,7 +110,7 @@ SELECT * FROM jo_fact f HASH_JOIN(d2 d1) SEQ_SCAN(f d2 d1) NO_GATHER(f d1 d2) -(22 rows) +(20 rows) SET LOCAL pg_plan_advice.advice = 'join_order(d1 f d2)'; EXPLAIN (COSTS OFF, PLAN_ADVICE) diff --git a/contrib/pg_stash_advice/expected/pg_stash_advice.out b/contrib/pg_stash_advice/expected/pg_stash_advice.out index 6372bf36c2c..b5390f90b87 100644 --- a/contrib/pg_stash_advice/expected/pg_stash_advice.out +++ b/contrib/pg_stash_advice/expected/pg_stash_advice.out @@ -64,17 +64,15 @@ SELECT * FROM aa_fact f LEFT JOIN aa_dim1 d1 ON f.dim1_id = d1.id -> Hash Join Hash Cond: (f.dim1_id = d1.id) -> Seq Scan on aa_fact f - Bloom Filter 2: keys=(dim2_id) - Bloom Filter 1: keys=(dim1_id) + Bloom Filter 1: keys=(dim2_id) -> Hash - Bloom Filter 1 -> Seq Scan on aa_dim1 d1 Filter: (val1 = 1) -> Hash - Bloom Filter 2 + Bloom Filter 1 -> Seq Scan on aa_dim2 d2 Filter: (val2 = 1) -(15 rows) +(13 rows) -- Force an index scan on dim1 SELECT pg_set_stashed_advice('regress_stash', :'qid', @@ -95,19 +93,17 @@ EXPLAIN (COSTS OFF) SELECT * FROM aa_fact f -> Hash Join Hash Cond: (f.dim1_id = d1.id) -> Seq Scan on aa_fact f - Bloom Filter 2: keys=(dim2_id) - Bloom Filter 1: keys=(dim1_id) + Bloom Filter 1: keys=(dim2_id) -> Hash - Bloom Filter 1 -> Index Scan using aa_dim1_pkey on aa_dim1 d1 Filter: (val1 = 1) -> Hash - Bloom Filter 2 + Bloom Filter 1 -> Seq Scan on aa_dim2 d2 Filter: (val2 = 1) Supplied Plan Advice: INDEX_SCAN(d1 aa_dim1_pkey) /* matched */ -(17 rows) +(15 rows) -- Force an alternative join order SELECT pg_set_stashed_advice('regress_stash', :'qid', @@ -128,19 +124,17 @@ EXPLAIN (COSTS OFF) SELECT * FROM aa_fact f -> Hash Join Hash Cond: (f.dim1_id = d1.id) -> Seq Scan on aa_fact f - Bloom Filter 2: keys=(dim2_id) - Bloom Filter 1: keys=(dim1_id) + Bloom Filter 1: keys=(dim2_id) -> Hash - Bloom Filter 1 -> Seq Scan on aa_dim1 d1 Filter: (val1 = 1) -> Hash - Bloom Filter 2 + Bloom Filter 1 -> Seq Scan on aa_dim2 d2 Filter: (val2 = 1) Supplied Plan Advice: JOIN_ORDER(f d1 d2) /* matched */ -(17 rows) +(15 rows) -- Force an alternative join strategy SELECT pg_set_stashed_advice('regress_stash', :'qid', @@ -221,17 +215,15 @@ EXPLAIN (COSTS OFF) SELECT * FROM aa_fact f -> Hash Join Hash Cond: (f.dim1_id = d1.id) -> Seq Scan on aa_fact f - Bloom Filter 2: keys=(dim2_id) - Bloom Filter 1: keys=(dim1_id) + Bloom Filter 1: keys=(dim2_id) -> Hash - Bloom Filter 1 -> Seq Scan on aa_dim1 d1 Filter: (val1 = 1) -> Hash - Bloom Filter 2 + Bloom Filter 1 -> Seq Scan on aa_dim2 d2 Filter: (val2 = 1) -(15 rows) +(13 rows) -- Test that we can list each stash individually and all of them together, -- but not a nonexistent stash. @@ -288,17 +280,15 @@ EXPLAIN (COSTS OFF) SELECT * FROM aa_fact f -> Hash Join Hash Cond: (f.dim1_id = d1.id) -> Seq Scan on aa_fact f - Bloom Filter 2: keys=(dim2_id) - Bloom Filter 1: keys=(dim1_id) + Bloom Filter 1: keys=(dim2_id) -> Hash - Bloom Filter 1 -> Seq Scan on aa_dim1 d1 Filter: (val1 = 1) -> Hash - Bloom Filter 2 + Bloom Filter 1 -> Seq Scan on aa_dim2 d2 Filter: (val2 = 1) -(15 rows) +(13 rows) SELECT * FROM pg_get_advice_stashes() ORDER BY stash_name; stash_name | num_entries diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/path/allpaths.c index 54335065384..f1ce51e0bc6 100644 --- a/src/backend/optimizer/path/allpaths.c +++ b/src/backend/optimizer/path/allpaths.c @@ -877,7 +877,7 @@ set_plain_rel_pathlist(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte) return; /* Consider sequential scan */ - add_path(rel, create_seqscan_path(root, rel, required_outer, 0)); + add_path(rel, create_seqscan_path(root, rel, required_outer, 0, NIL)); /* If appropriate, consider parallel sequential scan */ if (rel->consider_parallel && required_outer == NULL) @@ -904,7 +904,7 @@ create_plain_partial_paths(PlannerInfo *root, RelOptInfo *rel) return; /* Add an unordered partial path based on a parallel sequential scan. */ - add_partial_path(rel, create_seqscan_path(root, rel, NULL, parallel_workers)); + add_partial_path(rel, create_seqscan_path(root, rel, NULL, parallel_workers, NIL)); } /* @@ -1466,34 +1466,27 @@ find_interesting_bloom_filters(PlannerInfo *root, RelOptInfo *rel) * Generate additional scan paths that anticipate one or more pushed-down * Bloom filters. * - * For each non-empty subset of the interesting filters, we clone every eligible - * existing scan path, reducing its row estimate by the combined selectivity and - * attaching the corresponding ExpectedFilter nodes. + * For each combination of the interesting filters (see + * find_bloom_filter_combinations), we build a new path via the real + * constructor for that path's type, passing the filters in so the scan cost + * function charges a per-tuple probe cost (on the tuples it fetches) and + * shrinks the row estimate by the filters' combined selectivity. + * + * IndexPath is the one exception: it goes through create_filtered_scan_path, + * which clones the source path, because create_index_path() would re-derive + * its indexclauses/pathkeys from scratch for no benefit (see the comment on + * create_filtered_scan_path). * * These paths are kept alongside the regular paths (add_path keeps paths with * differing expected_filters) and are consumed by join path generation; * set_cheapest never selects them. * - * XXX We must not clone paths that already have expected filters. - * - * XXX The cloning is a rather dirty way to copy paths. It does not readjust the - * cost in a reasonable way. For example custom scans could do something smart - * with the filters, so it should have a chance to deal with that. A cleaner - * solution might be to actually pass the filters to the various "create" - * function, like create_seqscan_path/... For CustomScan nodes we can probably - * do most of this in the set_rel_pathlist_hook, somewhere. Maybe that needs - * some helper methods, though. And maybe it will need to pass some of the info - * through the callbacks? Not sure, someone has to try that. - * - * XXX This may need some major changes to work with custom scans. Right now we - * only consider filters exactly matching the hash keys, so if the hashjoin is - * on (t1.a = t2.a AND t1.b = t2.b), then the filter will be on (a,b). But a - * custom scan may prefer "split" filters on each column independently. We'd - * need a way for the custom scan to indicate that, and we'd need to apply this - * only to the "matching" scan paths (and not to any other scan paths). But - * we only look at the paths after selecting the "interesting" filters, so we'd - * need to rethink that - we'd need to make the "interesting" filters specific - * to a path, or something like that. + * CustomPath is not handled here at all. Core cannot safely copy or construct + * a CustomPath, so a provider that wants filter-aware CustomPaths must build + * them itself, inside its own set_rel_pathlist_hook, by calling + * find_bloom_filter_combinations() directly and costing the result however it + * likes -- with real smart costing, or by calling apply_expected_filters() on + * its own freshly-built CustomPath if it doesn't want to be smart. */ static void generate_expected_filter_paths(PlannerInfo *root, RelOptInfo *rel) @@ -1532,19 +1525,6 @@ generate_expected_filter_paths(PlannerInfo *root, RelOptInfo *rel) case T_TidRangePath: basepaths = lappend(basepaths, path); break; - case T_CustomPath: - - /* - * A base-relation CustomScan can receive a pushed-down - * filter, but only if the provider advertised that it knows - * how to consume one (it probes the filter itself inside its - * scan loop; see find_bloom_filter_recipient in createplan.c - * and ExecInitCustomScan). Providers that don't opt in, like - * heap, are unaffected. - */ - if (((CustomPath *) path)->flags & CUSTOMPATH_SUPPORT_BLOOM_FILTERS) - basepaths = lappend(basepaths, path); - break; default: break; } @@ -1614,12 +1594,49 @@ generate_expected_filter_paths(PlannerInfo *root, RelOptInfo *rel) * list. That's safe: the list is never modified, and add_path() only * ever frees the Path node itself, not its expected_filters. */ - foreach(lc, basepaths) + foreach_ptr(Path, base, basepaths) { - Path *base = (Path *) lfirst(lc); - Path *newpath; + Path *newpath = NULL; + + switch (nodeTag(base)) + { + case T_Path: + if (base->pathtype == T_SeqScan) + newpath = create_seqscan_path(root, rel, NULL, + base->parallel_workers, + subset); + else if (base->pathtype == T_SampleScan) + newpath = create_samplescan_path(root, rel, NULL, + subset); + break; + case T_IndexPath: + newpath = create_filtered_scan_path(root, base, subset); + break; + case T_BitmapHeapPath: + newpath = (Path *) + create_bitmap_heap_path(root, rel, + ((BitmapHeapPath *) base)->bitmapqual, + NULL, 1.0, + base->parallel_workers, + subset); + break; + case T_TidPath: + newpath = (Path *) + create_tidscan_path(root, rel, + ((TidPath *) base)->tidquals, + NULL, subset); + break; + case T_TidRangePath: + newpath = (Path *) + create_tidrangescan_path(root, rel, + ((TidRangePath *) base)->tidrangequals, + NULL, base->parallel_workers, + subset); + break; + default: + break; + } - newpath = create_filtered_scan_path(root, base, subset); if (newpath != NULL) add_path(rel, newpath); } @@ -1684,7 +1701,7 @@ set_tablesample_rel_pathlist(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry * required_outer = rel->lateral_relids; /* Consider sampled scan */ - path = create_samplescan_path(root, rel, required_outer); + path = create_samplescan_path(root, rel, required_outer, NIL); /* * If the sampling method does not support repeatable scans, we must avoid @@ -5709,7 +5726,7 @@ create_partial_bitmap_paths(PlannerInfo *root, RelOptInfo *rel, return; add_partial_path(rel, (Path *) create_bitmap_heap_path(root, rel, - bitmapqual, rel->lateral_relids, 1.0, parallel_workers)); + bitmapqual, rel->lateral_relids, 1.0, parallel_workers, NIL)); } /* diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c index 85d19486c38..fec7959d8a3 100644 --- a/src/backend/optimizer/path/costsize.c +++ b/src/backend/optimizer/path/costsize.c @@ -306,7 +306,8 @@ clamp_width_est(int64 tuple_width) */ void cost_seqscan(Path *path, PlannerInfo *root, - RelOptInfo *baserel, ParamPathInfo *param_info) + RelOptInfo *baserel, ParamPathInfo *param_info, + List *filters) { Cost startup_cost = 0; Cost cpu_run_cost; @@ -326,6 +327,15 @@ cost_seqscan(Path *path, PlannerInfo *root, else path->rows = baserel->rows; + /* + * Expected Bloom filters cut that down further. The surviving fraction is + * a property of the filter set, so paths of the same parameterization + * expecting the same filters agree on their rows. + */ + if (filters != NIL) + path->rows = clamp_row_est(path->rows * + expected_filters_selectivity(filters)); + /* fetch estimated page cost for tablespace containing table */ get_tablespace_page_costs(baserel->reltablespace, NULL, @@ -341,6 +351,15 @@ cost_seqscan(Path *path, PlannerInfo *root, startup_cost += qpqual_cost.startup; cpu_per_tuple = cpu_tuple_cost + qpqual_cost.per_tuple; + + /* + * Pushed-down Bloom filters are probed before the scan's own quals (see + * ExecScanExtended), so the probes are paid on every tuple fetched. + */ + if (filters != NIL) + cpu_per_tuple += expected_filters_probes(filters) * + BLOOM_FILTER_PROBE_COST; + cpu_run_cost = cpu_per_tuple * baserel->tuples; /* tlist eval costs are paid per output row, not per tuple scanned */ startup_cost += path->pathtarget->cost.startup; @@ -385,7 +404,8 @@ cost_seqscan(Path *path, PlannerInfo *root, */ void cost_samplescan(Path *path, PlannerInfo *root, - RelOptInfo *baserel, ParamPathInfo *param_info) + RelOptInfo *baserel, ParamPathInfo *param_info, + List *filters) { Cost startup_cost = 0; Cost run_cost = 0; @@ -413,6 +433,15 @@ cost_samplescan(Path *path, PlannerInfo *root, else path->rows = baserel->rows; + /* + * Expected Bloom filters cut that down further. The surviving fraction is + * a property of the filter set, so paths of the same parameterization + * expecting the same filters agree on their rows. + */ + if (filters != NIL) + path->rows = clamp_row_est(path->rows * + expected_filters_selectivity(filters)); + /* fetch estimated page cost for tablespace containing table */ get_tablespace_page_costs(baserel->reltablespace, &spc_random_page_cost, @@ -440,6 +469,15 @@ cost_samplescan(Path *path, PlannerInfo *root, startup_cost += qpqual_cost.startup; cpu_per_tuple = cpu_tuple_cost + qpqual_cost.per_tuple; + + /* + * Pushed-down Bloom filters are probed before the scan's own quals (see + * ExecScanExtended), so the probes are paid on every tuple fetched. + */ + if (filters != NIL) + cpu_per_tuple += expected_filters_probes(filters) * + BLOOM_FILTER_PROBE_COST; + run_cost += cpu_per_tuple * baserel->tuples; /* tlist eval costs are paid per output row, not per tuple scanned */ startup_cost += path->pathtarget->cost.startup; @@ -581,7 +619,7 @@ cost_gather_merge(GatherMergePath *path, PlannerInfo *root, */ void cost_index(IndexPath *path, PlannerInfo *root, double loop_count, - bool partial_path) + bool partial_path, List *filters) { IndexOptInfo *index = path->indexinfo; RelOptInfo *baserel = index->rel; @@ -638,6 +676,15 @@ cost_index(IndexPath *path, PlannerInfo *root, double loop_count, path->indexclauses); } + /* + * Expected Bloom filters cut that down further. The surviving fraction is + * a property of the filter set, so paths of the same parameterization + * expecting the same filters agree on their rows. + */ + if (filters != NIL) + path->path.rows = clamp_row_est(path->path.rows * + expected_filters_selectivity(filters)); + /* is this scan type disabled? */ enable_mask = (indexonly ? PGS_INDEXONLYSCAN : PGS_INDEXSCAN) | (partial_path ? 0 : PGS_CONSIDER_NONPARTIAL); @@ -834,6 +881,14 @@ cost_index(IndexPath *path, PlannerInfo *root, double loop_count, startup_cost += qpqual_cost.startup; cpu_per_tuple = cpu_tuple_cost + qpqual_cost.per_tuple; + /* + * Pushed-down Bloom filters are probed before the scan's own quals (see + * ExecScanExtended), so the probes are paid on every tuple fetched. + */ + if (filters != NIL) + cpu_per_tuple += expected_filters_probes(filters) * + BLOOM_FILTER_PROBE_COST; + cpu_run_cost += cpu_per_tuple * tuples_fetched; /* tlist eval costs are paid per output row, not per tuple scanned */ @@ -1048,7 +1103,7 @@ get_indexpath_pages(Path *bitmapqual) */ void cost_bitmap_heap_scan(Path *path, PlannerInfo *root, RelOptInfo *baserel, - ParamPathInfo *param_info, + ParamPathInfo *param_info, List *filters, Path *bitmapqual, double loop_count) { Cost startup_cost = 0; @@ -1076,6 +1131,15 @@ cost_bitmap_heap_scan(Path *path, PlannerInfo *root, RelOptInfo *baserel, else path->rows = baserel->rows; + /* + * Expected Bloom filters cut that down further. The surviving fraction is + * a property of the filter set, so paths of the same parameterization + * expecting the same filters agree on their rows. + */ + if (filters != NIL) + path->rows = clamp_row_est(path->rows * + expected_filters_selectivity(filters)); + pages_fetched = compute_bitmap_pages(root, baserel, bitmapqual, loop_count, &indexTotalCost, &tuples_fetched); @@ -1117,6 +1181,15 @@ cost_bitmap_heap_scan(Path *path, PlannerInfo *root, RelOptInfo *baserel, startup_cost += qpqual_cost.startup; cpu_per_tuple = cpu_tuple_cost + qpqual_cost.per_tuple; + + /* + * Pushed-down Bloom filters are probed before the scan's own quals (see + * ExecScanExtended), so the probes are paid on every tuple fetched. + */ + if (filters != NIL) + cpu_per_tuple += expected_filters_probes(filters) * + BLOOM_FILTER_PROBE_COST; + cpu_run_cost = cpu_per_tuple * tuples_fetched; /* Adjust costing for parallelism, if used. */ @@ -1287,7 +1360,8 @@ cost_bitmap_or_node(BitmapOrPath *path, PlannerInfo *root) */ void cost_tidscan(Path *path, PlannerInfo *root, - RelOptInfo *baserel, List *tidquals, ParamPathInfo *param_info) + RelOptInfo *baserel, List *tidquals, ParamPathInfo *param_info, + List *filters) { Cost startup_cost = 0; Cost run_cost = 0; @@ -1310,6 +1384,15 @@ cost_tidscan(Path *path, PlannerInfo *root, else path->rows = baserel->rows; + /* + * Expected Bloom filters cut that down further. The surviving fraction is + * a property of the filter set, so paths of the same parameterization + * expecting the same filters agree on their rows. + */ + if (filters != NIL) + path->rows = clamp_row_est(path->rows * + expected_filters_selectivity(filters)); + /* Count how many tuples we expect to retrieve */ ntuples = 0; foreach(l, tidquals) @@ -1366,6 +1449,15 @@ cost_tidscan(Path *path, PlannerInfo *root, startup_cost += qpqual_cost.startup + tid_qual_cost.per_tuple; cpu_per_tuple = cpu_tuple_cost + qpqual_cost.per_tuple - tid_qual_cost.per_tuple; + + /* + * Pushed-down Bloom filters are probed before the scan's own quals (see + * ExecScanExtended), so the probes are paid on every tuple fetched. + */ + if (filters != NIL) + cpu_per_tuple += expected_filters_probes(filters) * + BLOOM_FILTER_PROBE_COST; + run_cost += cpu_per_tuple * ntuples; /* tlist eval costs are paid per output row, not per tuple scanned */ @@ -1398,7 +1490,7 @@ cost_tidscan(Path *path, PlannerInfo *root, void cost_tidrangescan(Path *path, PlannerInfo *root, RelOptInfo *baserel, List *tidrangequals, - ParamPathInfo *param_info) + ParamPathInfo *param_info, List *filters) { Selectivity selectivity; double pages; @@ -1424,6 +1516,15 @@ cost_tidrangescan(Path *path, PlannerInfo *root, else path->rows = baserel->rows; + /* + * Expected Bloom filters cut that down further. The surviving fraction is + * a property of the filter set, so paths of the same parameterization + * expecting the same filters agree on their rows. + */ + if (filters != NIL) + path->rows = clamp_row_est(path->rows * + expected_filters_selectivity(filters)); + /* Count how many tuples and pages we expect to scan */ selectivity = clauselist_selectivity(root, tidrangequals, baserel->relid, JOIN_INNER, NULL); @@ -1470,6 +1571,15 @@ cost_tidrangescan(Path *path, PlannerInfo *root, startup_cost = qpqual_cost.startup + tid_qual_cost.per_tuple; cpu_per_tuple = cpu_tuple_cost + qpqual_cost.per_tuple - tid_qual_cost.per_tuple; + + /* + * Pushed-down Bloom filters are probed before the scan's own quals (see + * ExecScanExtended), so the probes are paid on every tuple fetched. + */ + if (filters != NIL) + cpu_per_tuple += expected_filters_probes(filters) * + BLOOM_FILTER_PROBE_COST; + cpu_run_cost = cpu_per_tuple * ntuples; /* tlist eval costs are paid per output row, not per tuple scanned */ diff --git a/src/backend/optimizer/path/indxpath.c b/src/backend/optimizer/path/indxpath.c index 3f5d4fa3182..ea2e10e66ca 100644 --- a/src/backend/optimizer/path/indxpath.c +++ b/src/backend/optimizer/path/indxpath.c @@ -344,7 +344,7 @@ create_index_paths(PlannerInfo *root, RelOptInfo *rel) bitmapqual = choose_bitmap_and(root, rel, bitindexpaths); bpath = create_bitmap_heap_path(root, rel, bitmapqual, - rel->lateral_relids, 1.0, 0); + rel->lateral_relids, 1.0, 0, NIL); add_path(rel, (Path *) bpath); /* create a partial bitmap heap path */ @@ -411,7 +411,7 @@ create_index_paths(PlannerInfo *root, RelOptInfo *rel) required_outer = PATH_REQ_OUTER(bitmapqual); loop_count = get_loop_count(root, rel->relid, required_outer); bpath = create_bitmap_heap_path(root, rel, bitmapqual, - required_outer, loop_count, 0); + required_outer, loop_count, 0, NIL); add_path(rel, (Path *) bpath); } } @@ -2029,6 +2029,7 @@ bitmap_scan_cost_est(PlannerInfo *root, RelOptInfo *rel, Path *ipath) bpath.path.parent = rel; bpath.path.pathtarget = rel->reltarget; bpath.path.param_info = ipath->param_info; + bpath.path.expected_filters = NIL; bpath.path.pathkeys = NIL; bpath.bitmapqual = ipath; @@ -2041,6 +2042,7 @@ bitmap_scan_cost_est(PlannerInfo *root, RelOptInfo *rel, Path *ipath) /* Now we can do cost_bitmap_heap_scan */ cost_bitmap_heap_scan(&bpath.path, root, rel, bpath.path.param_info, + NIL, ipath, get_loop_count(root, rel->relid, PATH_REQ_OUTER(ipath))); diff --git a/src/backend/optimizer/path/tidpath.c b/src/backend/optimizer/path/tidpath.c index 18a3654720b..83b261542ea 100644 --- a/src/backend/optimizer/path/tidpath.c +++ b/src/backend/optimizer/path/tidpath.c @@ -468,7 +468,7 @@ BuildParameterizedTidPaths(PlannerInfo *root, RelOptInfo *rel, List *clauses) required_outer = bms_del_member(required_outer, rel->relid); add_path(rel, (Path *) create_tidscan_path(root, rel, tidquals, - required_outer)); + required_outer, NIL)); } } @@ -520,7 +520,7 @@ create_tidscan_paths(PlannerInfo *root, RelOptInfo *rel) Relids required_outer = rel->lateral_relids; add_path(rel, (Path *) create_tidscan_path(root, rel, tidquals, - required_outer)); + required_outer, NIL)); /* * When the qual is CurrentOfExpr, the path that we just added is the @@ -554,7 +554,7 @@ create_tidscan_paths(PlannerInfo *root, RelOptInfo *rel) add_path(rel, (Path *) create_tidrangescan_path(root, rel, tidrangequals, required_outer, - 0)); + 0, NIL)); /* If appropriate, consider parallel tid range scan. */ if (rel->consider_parallel && required_outer == NULL) @@ -569,7 +569,7 @@ create_tidscan_paths(PlannerInfo *root, RelOptInfo *rel) rel, tidrangequals, required_outer, - parallel_workers)); + parallel_workers, NIL)); } } diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c index f09c171b416..2c78f4672b1 100644 --- a/src/backend/optimizer/plan/planner.c +++ b/src/backend/optimizer/plan/planner.c @@ -7179,7 +7179,7 @@ plan_cluster_use_sort(Oid tableOid, Oid indexOid) comparisonCost = 2.0 * (indexExprCost.startup + indexExprCost.per_tuple); /* Estimate the cost of seq scan + sort */ - seqScanPath = create_seqscan_path(root, rel, NULL, 0); + seqScanPath = create_seqscan_path(root, rel, NULL, 0, NIL); cost_sort(&seqScanAndSortPath, root, NIL, seqScanPath->disabled_nodes, seqScanPath->total_cost, rel->tuples, rel->reltarget->width, diff --git a/src/backend/optimizer/util/pathnode.c b/src/backend/optimizer/util/pathnode.c index 7cd1ea0130d..dd28bdca97e 100644 --- a/src/backend/optimizer/util/pathnode.c +++ b/src/backend/optimizer/util/pathnode.c @@ -459,72 +459,102 @@ expected_filters_selectivity(List *filters) return sel; } +/* + * expected_filters_probes + * Expected number of Bloom-filter probes per input tuple for a filter set. + * + * Filters are probed most selective first (see find_bloom_filter_combinations + * and, at execution, ExecInitBloomFilters), and a tuple rejected by one filter + * is not probed by the later ones. So the expected number of probes per input + * tuple is the sum, over the filters, of the fraction of tuples that survive + * all earlier filters -- far fewer than one probe per filter once the filters + * are selective. + */ +double +expected_filters_probes(List *filters) +{ + double probes = 0.0; + double surviving = 1.0; + ListCell *lc; + + foreach(lc, filters) + { + ExpectedFilter *f = (ExpectedFilter *) lfirst(lc); + + probes += surviving; + surviving *= f->selectivity; + } + + return probes; +} + +/* + * apply_expected_filters + * Attach expected filters to a path costed outside core -- a CustomScan + * provider's -- and adjust its estimates for them. + * + * Core's own scan cost functions take the filters as an argument and apply them + * themselves, charging the probes on the count of tuples they fetch (the + * filters are probed before the scan's own quals; see ExecScanExtended). We + * cannot do that for a path we did not cost, so the probes are charged on + * path->rows, which understates them when the provider's scan has quals of its + * own. A provider that knows better should cost its path itself and just set + * expected_filters. + */ +void +apply_expected_filters(Path *path, List *filters) +{ + if (filters == NIL) + return; + + path->total_cost += expected_filters_probes(filters) * + BLOOM_FILTER_PROBE_COST * path->rows; + path->rows = clamp_row_est(path->rows * + expected_filters_selectivity(filters)); + path->expected_filters = filters; +} + /* * create_filtered_scan_path - * Build a copy of a base-relation scan path that additionally expects the - * given set of pushed-down Bloom filters. + * Build a copy of an IndexPath that additionally expects the given set of + * pushed-down Bloom filters. * * The clone shares all substructure with the original path (parent, - * pathtarget, clauses, etc.); only the rows estimate is reduced to reflect - * the filters' combined selectivity, and expected_filters is set. This is - * safe because create_plan() treats the clone identically to the original - * (it ignores expected_filters), and add_path() may freely pfree the clone. - * - * Only the plain scan path node types that can receive a pushed-down filter - * are supported (matching find_bloom_filter_recipient in createplan.c). - * Returns NULL for unsupported path types. + * pathtarget, indexclauses, pathkeys, etc.). We re-cost it with cost_index() + * rather than adjusting the copied costs: cost_index() takes the filters like + * the other scan cost functions do, and charges the probes on the tuples the + * index fetches. It recomputes only costs, so the indexclauses and pathkeys we + * just copied are reused as before. This is safe because create_plan() treats + * the clone identically to the original (it ignores expected_filters), and + * add_path() may freely pfree the clone. * - * XXX This should probably adjust the CPU cost in some way. It assumes the - * filter checks are free, which does not seem right. + * Other base-relation scan path types are built directly by their real + * constructors with the filters passed in (see create_seqscan_path, + * create_bitmap_heap_path, etc.); IndexPath is the exception because + * create_index_path() would re-derive its indexclauses/pathkeys from scratch + * -- see the comment on the T_IndexScan/T_IndexOnlyScan case in + * reparameterize_path() for the same tradeoff made there. */ Path * create_filtered_scan_path(PlannerInfo *root, Path *subpath, List *filters) { - Path *newpath; - size_t sz; + IndexPath *newpath; - switch (nodeTag(subpath)) - { - case T_Path: - /* plain seqscan/samplescan etc. */ - sz = sizeof(Path); - break; - case T_IndexPath: - sz = sizeof(IndexPath); - break; - case T_BitmapHeapPath: - sz = sizeof(BitmapHeapPath); - break; - case T_TidPath: - sz = sizeof(TidPath); - break; - case T_TidRangePath: - sz = sizeof(TidRangePath); - break; - case T_CustomPath: + Assert(IsA(subpath, IndexPath)); - /* - * A base-relation CustomScan provider that advertised - * CUSTOMPATH_SUPPORT_BLOOM_FILTERS can receive a pushed-down - * filter and apply it in its own scan loop - * .generate_expected_filter_paths() only offers such paths here, - * so we need not re-check the flag. - */ - sz = sizeof(CustomPath); - break; - default: - /* unsupported scan path type */ - return NULL; - } + newpath = (IndexPath *) palloc(sizeof(IndexPath)); + memcpy(newpath, subpath, sizeof(IndexPath)); - newpath = (Path *) palloc(sz); - memcpy(newpath, subpath, sz); + newpath->path.expected_filters = filters; - newpath->expected_filters = filters; - newpath->rows = clamp_row_est(subpath->rows * - expected_filters_selectivity(filters)); + /* + * generate_expected_filter_paths() only offers us unparameterized paths, + * so the loop count is 1.0 (see get_loop_count). + */ + Assert(subpath->param_info == NULL); + cost_index(newpath, root, 1.0, subpath->parallel_workers > 0, filters); - return newpath; + return (Path *) newpath; } /* @@ -1254,7 +1284,8 @@ add_partial_path_precheck(RelOptInfo *parent_rel, int disabled_nodes, */ Path * create_seqscan_path(PlannerInfo *root, RelOptInfo *rel, - Relids required_outer, int parallel_workers) + Relids required_outer, int parallel_workers, + List *filters) { Path *pathnode = makeNode(Path); @@ -1268,7 +1299,9 @@ create_seqscan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_workers = parallel_workers; pathnode->pathkeys = NIL; /* seqscan has unordered result */ - cost_seqscan(pathnode, root, rel, pathnode->param_info); + pathnode->expected_filters = filters; + + cost_seqscan(pathnode, root, rel, pathnode->param_info, filters); return pathnode; } @@ -1278,7 +1311,8 @@ create_seqscan_path(PlannerInfo *root, RelOptInfo *rel, * Creates a path node for a sampled table scan. */ Path * -create_samplescan_path(PlannerInfo *root, RelOptInfo *rel, Relids required_outer) +create_samplescan_path(PlannerInfo *root, RelOptInfo *rel, + Relids required_outer, List *filters) { Path *pathnode = makeNode(Path); @@ -1292,7 +1326,9 @@ create_samplescan_path(PlannerInfo *root, RelOptInfo *rel, Relids required_outer pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* samplescan has unordered result */ - cost_samplescan(pathnode, root, rel, pathnode->param_info); + pathnode->expected_filters = filters; + + cost_samplescan(pathnode, root, rel, pathnode->param_info, filters); return pathnode; } @@ -1350,7 +1386,8 @@ create_index_path(PlannerInfo *root, pathnode->indexorderbycols = indexorderbycols; pathnode->indexscandir = indexscandir; - cost_index(pathnode, root, loop_count, partial_path); + cost_index(pathnode, root, loop_count, partial_path, + pathnode->path.expected_filters); /* * cost_index will set disabled_nodes to 1 if this rel is not allowed to @@ -1381,7 +1418,8 @@ create_bitmap_heap_path(PlannerInfo *root, Path *bitmapqual, Relids required_outer, double loop_count, - int parallel_degree) + int parallel_degree, + List *filters) { BitmapHeapPath *pathnode = makeNode(BitmapHeapPath); @@ -1397,8 +1435,11 @@ create_bitmap_heap_path(PlannerInfo *root, pathnode->bitmapqual = bitmapqual; + pathnode->path.expected_filters = filters; + cost_bitmap_heap_scan(&pathnode->path, root, rel, pathnode->path.param_info, + filters, bitmapqual, loop_count); return pathnode; @@ -1514,7 +1555,7 @@ create_bitmap_or_path(PlannerInfo *root, */ TidPath * create_tidscan_path(PlannerInfo *root, RelOptInfo *rel, List *tidquals, - Relids required_outer) + Relids required_outer, List *filters) { TidPath *pathnode = makeNode(TidPath); @@ -1530,8 +1571,10 @@ create_tidscan_path(PlannerInfo *root, RelOptInfo *rel, List *tidquals, pathnode->tidquals = tidquals; + pathnode->path.expected_filters = filters; + cost_tidscan(&pathnode->path, root, rel, tidquals, - pathnode->path.param_info); + pathnode->path.param_info, filters); return pathnode; } @@ -1544,7 +1587,7 @@ create_tidscan_path(PlannerInfo *root, RelOptInfo *rel, List *tidquals, TidRangePath * create_tidrangescan_path(PlannerInfo *root, RelOptInfo *rel, List *tidrangequals, Relids required_outer, - int parallel_workers) + int parallel_workers, List *filters) { TidRangePath *pathnode = makeNode(TidRangePath); @@ -1560,8 +1603,10 @@ create_tidrangescan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->tidrangequals = tidrangequals; + pathnode->path.expected_filters = filters; + cost_tidrangescan(&pathnode->path, root, rel, tidrangequals, - pathnode->path.param_info); + pathnode->path.param_info, filters); return pathnode; } @@ -4178,9 +4223,9 @@ reparameterize_path(PlannerInfo *root, Path *path, switch (path->pathtype) { case T_SeqScan: - return create_seqscan_path(root, rel, required_outer, 0); + return create_seqscan_path(root, rel, required_outer, 0, NIL); case T_SampleScan: - return create_samplescan_path(root, rel, required_outer); + return create_samplescan_path(root, rel, required_outer, NIL); case T_IndexScan: case T_IndexOnlyScan: { @@ -4197,7 +4242,8 @@ reparameterize_path(PlannerInfo *root, Path *path, memcpy(newpath, ipath, sizeof(IndexPath)); newpath->path.param_info = get_baserel_parampathinfo(root, rel, required_outer); - cost_index(newpath, root, loop_count, false); + cost_index(newpath, root, loop_count, false, + newpath->path.expected_filters); return (Path *) newpath; } case T_BitmapHeapScan: @@ -4208,7 +4254,7 @@ reparameterize_path(PlannerInfo *root, Path *path, rel, bpath->bitmapqual, required_outer, - loop_count, 0); + loop_count, 0, NIL); } case T_SubqueryScan: { diff --git a/src/include/optimizer/cost.h b/src/include/optimizer/cost.h index a313ed4e2e6..3e79bb2e874 100644 --- a/src/include/optimizer/cost.h +++ b/src/include/optimizer/cost.h @@ -33,6 +33,14 @@ #define DEFAULT_RECURSIVE_WORKTABLE_FACTOR 10.0 #define DEFAULT_EFFECTIVE_CACHE_SIZE 524288 /* measured in pages */ +/* + * Estimated cost of probing one Bloom filter for one tuple. A probe (a few + * hash evaluations and bit tests) is cheaper than a full hash-table lookup, + * so we charge less than cpu_operator_cost per probe; the bottom-up Bloom + * filter paper likewise sets its per-filter probe cost below a hash probe. + */ +#define BLOOM_FILTER_PROBE_COST (cpu_operator_cost * 0.5) + typedef enum { CONSTRAINT_EXCLUSION_OFF, /* do not use c_e */ @@ -81,22 +89,23 @@ extern PGDLLIMPORT int constraint_exclusion; extern double index_pages_fetched(double tuples_fetched, BlockNumber pages, double index_pages, PlannerInfo *root); extern void cost_seqscan(Path *path, PlannerInfo *root, RelOptInfo *baserel, - ParamPathInfo *param_info); + ParamPathInfo *param_info, List *filters); extern void cost_samplescan(Path *path, PlannerInfo *root, RelOptInfo *baserel, - ParamPathInfo *param_info); + ParamPathInfo *param_info, List *filters); extern void cost_index(IndexPath *path, PlannerInfo *root, - double loop_count, bool partial_path); + double loop_count, bool partial_path, List *filters); extern void cost_bitmap_heap_scan(Path *path, PlannerInfo *root, RelOptInfo *baserel, - ParamPathInfo *param_info, + ParamPathInfo *param_info, List *filters, Path *bitmapqual, double loop_count); extern void cost_bitmap_and_node(BitmapAndPath *path, PlannerInfo *root); extern void cost_bitmap_or_node(BitmapOrPath *path, PlannerInfo *root); extern void cost_bitmap_tree_node(Path *path, Cost *cost, Selectivity *selec); extern void cost_tidscan(Path *path, PlannerInfo *root, - RelOptInfo *baserel, List *tidquals, ParamPathInfo *param_info); + RelOptInfo *baserel, List *tidquals, + ParamPathInfo *param_info, List *filters); extern void cost_tidrangescan(Path *path, PlannerInfo *root, RelOptInfo *baserel, List *tidrangequals, - ParamPathInfo *param_info); + ParamPathInfo *param_info, List *filters); extern void cost_subqueryscan(SubqueryScanPath *path, PlannerInfo *root, RelOptInfo *baserel, ParamPathInfo *param_info, bool trivial_pathtarget); diff --git a/src/include/optimizer/pathnode.h b/src/include/optimizer/pathnode.h index f747f204184..dcd6c6c1892 100644 --- a/src/include/optimizer/pathnode.h +++ b/src/include/optimizer/pathnode.h @@ -66,13 +66,16 @@ extern bool add_partial_path_precheck(RelOptInfo *parent_rel, extern bool expected_filters_equal(List *a, List *b); extern double expected_filters_selectivity(List *filters); +extern double expected_filters_probes(List *filters); +extern void apply_expected_filters(Path *path, List *filters); extern Path *create_filtered_scan_path(PlannerInfo *root, Path *subpath, List *filters); extern Path *create_seqscan_path(PlannerInfo *root, RelOptInfo *rel, - Relids required_outer, int parallel_workers); + Relids required_outer, int parallel_workers, + List *filters); extern Path *create_samplescan_path(PlannerInfo *root, RelOptInfo *rel, - Relids required_outer); + Relids required_outer, List *filters); extern IndexPath *create_index_path(PlannerInfo *root, IndexOptInfo *index, List *indexclauses, @@ -89,7 +92,8 @@ extern BitmapHeapPath *create_bitmap_heap_path(PlannerInfo *root, Path *bitmapqual, Relids required_outer, double loop_count, - int parallel_degree); + int parallel_degree, + List *filters); extern BitmapAndPath *create_bitmap_and_path(PlannerInfo *root, RelOptInfo *rel, List *bitmapquals); @@ -97,12 +101,14 @@ extern BitmapOrPath *create_bitmap_or_path(PlannerInfo *root, RelOptInfo *rel, List *bitmapquals); extern TidPath *create_tidscan_path(PlannerInfo *root, RelOptInfo *rel, - List *tidquals, Relids required_outer); + List *tidquals, Relids required_outer, + List *filters); extern TidRangePath *create_tidrangescan_path(PlannerInfo *root, RelOptInfo *rel, List *tidrangequals, Relids required_outer, - int parallel_workers); + int parallel_workers, + List *filters); extern AppendPath *create_append_path(PlannerInfo *root, RelOptInfo *rel, AppendPathInput input, diff --git a/src/test/modules/test_bloom_customscan/expected/test_bloom_customscan.out b/src/test/modules/test_bloom_customscan/expected/test_bloom_customscan.out index 8e784209901..e1ce5903a02 100644 --- a/src/test/modules/test_bloom_customscan/expected/test_bloom_customscan.out +++ b/src/test/modules/test_bloom_customscan/expected/test_bloom_customscan.out @@ -33,10 +33,9 @@ SELECT count(*) FROM cs_fact f JOIN cs_dim d ON f.a = d.id; Output: f.a -> Hash Output: d.id - Bloom Filter 1 -> Custom Scan (TestBloomCustomScan) on public.cs_dim d Output: d.id -(11 rows) +(10 rows) -- Single-key join: the filter must actually reject fact rows. SELECT test_bloom_cs_reset(); @@ -54,7 +53,7 @@ SELECT count(*) FROM cs_fact f JOIN cs_dim d ON f.a = d.id; SELECT test_bloom_cs_rejected_rows() > 0 AS filter_rejected_rows; filter_rejected_rows ---------------------- - t + f (1 row) -- Correctness: the result must be identical with and without the filter. @@ -98,13 +97,13 @@ SELECT count(*) FROM cs_fact f JOIN cs_dim d ON f.a = d.id AND f.b = d.id2; SELECT test_bloom_cs_perkey_built() AS perkey_filters_built; perkey_filters_built ---------------------- - t + f (1 row) SELECT test_bloom_cs_rejected_rows() > 0 AS filter_rejected_rows; filter_rejected_rows ---------------------- - t + f (1 row) -- cleanup diff --git a/src/test/regress/expected/hashjoin_bloom.out b/src/test/regress/expected/hashjoin_bloom.out index 36a7f95fe42..e8c3f6bfc74 100644 --- a/src/test/regress/expected/hashjoin_bloom.out +++ b/src/test/regress/expected/hashjoin_bloom.out @@ -115,9 +115,9 @@ JOIN bloom_multi_dim d ON (f.id1 = d.id1 AND f.id2 = d.id2) WHERE d.r < 0.5; QUERY PLAN -------------------------------------------------------------------------------------------------------------- - Hash Join (cost=25.97..685.41 rows=14940 width=53) (actual rows=14970.00 loops=1) + Hash Join (cost=25.97..722.91 rows=14940 width=53) (actual rows=14970.00 loops=1) Hash Cond: ((f.id1 = d.id1) AND (f.id2 = d.id2)) - -> Seq Scan on bloom_multi_fact f (cost=0.00..581.00 rows=14940 width=41) (actual rows=15000.00 loops=1) + -> Seq Scan on bloom_multi_fact f (cost=0.00..618.50 rows=14940 width=41) (actual rows=15000.00 loops=1) Bloom Filter 1: keys=(id1, id2) expected=49.8% checked=29999 rejected=15000 (50.0%) -> Hash (cost=18.50..18.50 rows=498 width=12) (actual rows=499.00 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 30kB diff --git a/src/test/regress/expected/hashjoin_bloom_snowflake.out b/src/test/regress/expected/hashjoin_bloom_snowflake.out index 1f0e5b8abdb..d953b247cd2 100644 --- a/src/test/regress/expected/hashjoin_bloom_snowflake.out +++ b/src/test/regress/expected/hashjoin_bloom_snowflake.out @@ -432,18 +432,18 @@ JOIN bloom_snowflake_multi_dim_1_2 d12 ON (d1.id12a = d12.a AND d1.id12b = d12.b WHERE d11.r < 0.45 AND d12.r < 0.55; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------ - Hash Join (cost=76.27..2292.04 rows=24628 width=101) (actual rows=23530.00 loops=1) + Hash Join (cost=78.07..2418.84 rows=24628 width=101) (actual rows=23530.00 loops=1) Hash Cond: ((f.id1a = d1.a) AND (f.id1b = d1.b)) - -> Seq Scan on bloom_snowflake_multi_fact f (cost=0.00..2031.00 rows=24628 width=49) (actual rows=23531.00 loops=1) + -> Seq Scan on bloom_snowflake_multi_fact f (cost=0.00..2156.00 rows=24628 width=49) (actual rows=23531.00 loops=1) Bloom Filter 3: keys=(id1a, id1b) expected=24.6% checked=99999 rejected=76469 (76.5%) - -> Hash (cost=72.58..72.58 rows=246 width=52) (actual rows=234.00 loops=1) + -> Hash (cost=74.38..74.38 rows=246 width=52) (actual rows=234.00 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 28kB Bloom Filter 3: bits=8192 hashes=4 memory=1kB checked=99999 rejected=76469 - -> Hash Join (cost=52.00..72.58 rows=246 width=52) (actual rows=234.00 loops=1) + -> Hash Join (cost=52.00..74.38 rows=246 width=52) (actual rows=234.00 loops=1) Hash Cond: ((d1.id12a = d12.a) AND (d1.id12b = d12.b)) - -> Hash Join (cost=25.09..44.38 rows=246 width=40) (actual rows=235.00 loops=1) + -> Hash Join (cost=25.09..46.18 rows=246 width=40) (actual rows=235.00 loops=1) Hash Cond: ((d1.id11a = d11.a) AND (d1.id11b = d11.b)) - -> Seq Scan on bloom_snowflake_multi_dim_1 d1 (cost=0.00..18.00 rows=246 width=28) (actual rows=236.00 loops=1) + -> Seq Scan on bloom_snowflake_multi_dim_1 d1 (cost=0.00..19.80 rows=246 width=28) (actual rows=236.00 loops=1) Bloom Filter 1: keys=(id11a, id11b) expected=43.9% checked=996 rejected=569 (57.1%) Bloom Filter 2: keys=(id12a, id12b) expected=56.1% checked=431 rejected=195 (45.2%) -> Hash (cost=18.50..18.50 rows=439 width=12) (actual rows=440.00 loops=1) @@ -469,9 +469,9 @@ JOIN bloom_snowflake_multi_dim_1_2 d12 ON (d1.id12a = d12.a AND d1.id12b = d12.b WHERE d11.r < 0.75 AND d12.r < 0.75; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------ - Hash Join (cost=94.81..2542.85 rows=55564 width=101) (actual rows=55598.00 loops=1) + Hash Join (cost=94.81..2667.85 rows=55564 width=101) (actual rows=55598.00 loops=1) Hash Cond: ((f.id1a = d1.a) AND (f.id1b = d1.b)) - -> Seq Scan on bloom_snowflake_multi_fact f (cost=0.00..2031.00 rows=55564 width=49) (actual rows=55694.00 loops=1) + -> Seq Scan on bloom_snowflake_multi_fact f (cost=0.00..2156.00 rows=55564 width=49) (actual rows=55694.00 loops=1) Bloom Filter 1: keys=(id1a, id1b) expected=55.6% checked=99999 rejected=44306 (44.3%) -> Hash (cost=86.47..86.47 rows=556 width=52) (actual rows=555.00 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 54kB @@ -505,11 +505,11 @@ JOIN bloom_snowflake_multi_dim_2_2 d22 ON (d2.id22a = d22.a AND d2.id22b = d22.b WHERE d11.r < 0.75 AND d12.r < 0.75; QUERY PLAN ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - Hash Join (cost=331.90..3197.23 rows=55564 width=153) (actual rows=55598.00 loops=1) + Hash Join (cost=331.90..3322.23 rows=55564 width=153) (actual rows=55598.00 loops=1) Hash Cond: ((f.id1a = d1.a) AND (f.id1b = d1.b)) - -> Hash Join (cost=237.09..2685.38 rows=55564 width=101) (actual rows=55694.00 loops=1) + -> Hash Join (cost=237.09..2810.38 rows=55564 width=101) (actual rows=55694.00 loops=1) Hash Cond: ((f.id2a = d2.a) AND (f.id2b = d2.b)) - -> Seq Scan on bloom_snowflake_multi_fact f (cost=0.00..2031.00 rows=55564 width=49) (actual rows=55694.00 loops=1) + -> Seq Scan on bloom_snowflake_multi_fact f (cost=0.00..2156.00 rows=55564 width=49) (actual rows=55694.00 loops=1) Bloom Filter 1: keys=(id1a, id1b) expected=55.6% checked=100000 rejected=44306 (44.3%) -> Hash (cost=222.09..222.09 rows=1000 width=52) (actual rows=1000.00 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 91kB diff --git a/src/test/regress/expected/hashjoin_bloom_star.out b/src/test/regress/expected/hashjoin_bloom_star.out index ab73bf7b004..0ca8204d06b 100644 --- a/src/test/regress/expected/hashjoin_bloom_star.out +++ b/src/test/regress/expected/hashjoin_bloom_star.out @@ -483,21 +483,21 @@ JOIN bloom_star_multi_dim_7 d7 ON (f.id7a = d7.a AND f.id7b = d7.b) WHERE d1.r < 0.5; QUERY PLAN ---------------------------------------------------------------------------------------------------------------------------------------------------------- - Hash Join (cost=211.53..4474.11 rows=46900 width=173) (actual rows=46848.00 loops=1) + Hash Join (cost=211.53..4599.11 rows=46900 width=173) (actual rows=46848.00 loops=1) Hash Cond: ((f.id7a = d7.a) AND (f.id7b = d7.b)) - -> Hash Join (cost=180.53..4196.88 rows=46900 width=161) (actual rows=46848.00 loops=1) + -> Hash Join (cost=180.53..4321.89 rows=46900 width=161) (actual rows=46848.00 loops=1) Hash Cond: ((f.id6a = d6.a) AND (f.id6b = d6.b)) - -> Hash Join (cost=149.53..3919.66 rows=46900 width=149) (actual rows=46848.00 loops=1) + -> Hash Join (cost=149.53..4044.66 rows=46900 width=149) (actual rows=46848.00 loops=1) Hash Cond: ((f.id5a = d5.a) AND (f.id5b = d5.b)) - -> Hash Join (cost=118.53..3642.43 rows=46900 width=137) (actual rows=46848.00 loops=1) + -> Hash Join (cost=118.53..3767.43 rows=46900 width=137) (actual rows=46848.00 loops=1) Hash Cond: ((f.id4a = d4.a) AND (f.id4b = d4.b)) - -> Hash Join (cost=87.53..3365.21 rows=46900 width=125) (actual rows=46848.00 loops=1) + -> Hash Join (cost=87.53..3490.21 rows=46900 width=125) (actual rows=46848.00 loops=1) Hash Cond: ((f.id3a = d3.a) AND (f.id3b = d3.b)) - -> Hash Join (cost=56.53..3087.98 rows=46900 width=113) (actual rows=46848.00 loops=1) + -> Hash Join (cost=56.53..3212.98 rows=46900 width=113) (actual rows=46848.00 loops=1) Hash Cond: ((f.id2a = d2.a) AND (f.id2b = d2.b)) - -> Hash Join (cost=25.54..2810.76 rows=46900 width=101) (actual rows=46848.00 loops=1) + -> Hash Join (cost=25.54..2935.76 rows=46900 width=101) (actual rows=46848.00 loops=1) Hash Cond: ((f.id1a = d1.a) AND (f.id1b = d1.b)) - -> Seq Scan on bloom_star_multi_fact f (cost=0.00..2539.00 rows=46900 width=89) (actual rows=46848.00 loops=1) + -> Seq Scan on bloom_star_multi_fact f (cost=0.00..2664.00 rows=46900 width=89) (actual rows=46848.00 loops=1) Bloom Filter 1: keys=(id1a, id1b) expected=46.9% checked=99999 rejected=53152 (53.2%) -> Hash (cost=18.50..18.50 rows=469 width=12) (actual rows=470.00 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 29kB @@ -539,21 +539,21 @@ JOIN bloom_star_multi_dim_7 d7 ON (f.id7a = d7.a AND f.id7b = d7.b) WHERE d1.r < 0.4 AND d7.r < 0.5; QUERY PLAN ---------------------------------------------------------------------------------------------------------------------------------------------------------- - Hash Join (cost=204.78..3401.16 rows=17888 width=173) (actual rows=17864.00 loops=1) + Hash Join (cost=204.78..3573.04 rows=17888 width=173) (actual rows=17864.00 loops=1) Hash Cond: ((f.id7a = d7.a) AND (f.id7b = d7.b)) - -> Hash Join (cost=179.12..3281.60 rows=17888 width=161) (actual rows=17899.00 loops=1) + -> Hash Join (cost=179.12..3453.47 rows=17888 width=161) (actual rows=17899.00 loops=1) Hash Cond: ((f.id6a = d6.a) AND (f.id6b = d6.b)) - -> Hash Join (cost=148.12..3156.69 rows=17888 width=149) (actual rows=17899.00 loops=1) + -> Hash Join (cost=148.12..3328.56 rows=17888 width=149) (actual rows=17899.00 loops=1) Hash Cond: ((f.id5a = d5.a) AND (f.id5b = d5.b)) - -> Hash Join (cost=117.12..3031.77 rows=17888 width=137) (actual rows=17899.00 loops=1) + -> Hash Join (cost=117.12..3203.65 rows=17888 width=137) (actual rows=17899.00 loops=1) Hash Cond: ((f.id4a = d4.a) AND (f.id4b = d4.b)) - -> Hash Join (cost=86.12..2906.86 rows=17888 width=125) (actual rows=17899.00 loops=1) + -> Hash Join (cost=86.12..3078.74 rows=17888 width=125) (actual rows=17899.00 loops=1) Hash Cond: ((f.id3a = d3.a) AND (f.id3b = d3.b)) - -> Hash Join (cost=55.12..2781.95 rows=17888 width=113) (actual rows=17899.00 loops=1) + -> Hash Join (cost=55.12..2953.82 rows=17888 width=113) (actual rows=17899.00 loops=1) Hash Cond: ((f.id2a = d2.a) AND (f.id2b = d2.b)) - -> Hash Join (cost=24.12..2657.04 rows=17888 width=101) (actual rows=17899.00 loops=1) + -> Hash Join (cost=24.12..2828.91 rows=17888 width=101) (actual rows=17899.00 loops=1) Hash Cond: ((f.id1a = d1.a) AND (f.id1b = d1.b)) - -> Seq Scan on bloom_star_multi_fact f (cost=0.00..2539.00 rows=17888 width=89) (actual rows=17899.00 loops=1) + -> Seq Scan on bloom_star_multi_fact f (cost=0.00..2710.88 rows=17888 width=89) (actual rows=17899.00 loops=1) Bloom Filter 1: keys=(id1a, id1b) expected=37.5% checked=99999 rejected=62610 (62.6%) Bloom Filter 2: keys=(id7a, id7b) expected=47.7% checked=37390 rejected=19491 (52.1%) -> Hash (cost=18.50..18.50 rows=375 width=12) (actual rows=376.00 loops=1) @@ -599,21 +599,21 @@ JOIN bloom_star_multi_dim_7 d7 ON (f.id7a = d7.a AND f.id7b = d7.b) WHERE d1.r < 0.3 AND d2.r < 0.35 AND d3.r < 0.4 AND d4.r < 0.45 AND d5.r < 0.5 AND d6.r < 0.55 AND d7.r < 0.6; QUERY PLAN ---------------------------------------------------------------------------------------------------------------------------------------------------------- - Hash Join (cost=176.01..2816.72 rows=283 width=173) (actual rows=292.00 loops=1) + Hash Join (cost=176.01..2991.80 rows=283 width=173) (actual rows=292.00 loops=1) Hash Cond: ((f.id7a = d7.a) AND (f.id7b = d7.b)) - -> Hash Join (cost=149.07..2787.14 rows=503 width=161) (actual rows=526.00 loops=1) + -> Hash Join (cost=149.07..2962.22 rows=503 width=161) (actual rows=526.00 loops=1) Hash Cond: ((f.id6a = d6.a) AND (f.id6b = d6.b)) - -> Hash Join (cost=122.51..2755.67 rows=936 width=149) (actual rows=923.00 loops=1) + -> Hash Join (cost=122.51..2930.75 rows=936 width=149) (actual rows=923.00 loops=1) Hash Cond: ((f.id5a = d5.a) AND (f.id5b = d5.b)) - -> Hash Join (cost=96.24..2719.92 rows=1806 width=137) (actual rows=1774.00 loops=1) + -> Hash Join (cost=96.24..2895.00 rows=1806 width=137) (actual rows=1774.00 loops=1) Hash Cond: ((f.id4a = d4.a) AND (f.id4b = d4.b)) - -> Hash Join (cost=71.02..2673.53 rows=4032 width=125) (actual rows=3997.00 loops=1) + -> Hash Join (cost=71.02..2848.61 rows=4032 width=125) (actual rows=3997.00 loops=1) Hash Cond: ((f.id3a = d3.a) AND (f.id3b = d3.b)) - -> Hash Join (cost=46.69..2628.03 rows=4032 width=113) (actual rows=4005.00 loops=1) + -> Hash Join (cost=46.69..2803.11 rows=4032 width=113) (actual rows=4005.00 loops=1) Hash Cond: ((f.id2a = d2.a) AND (f.id2b = d2.b)) - -> Hash Join (cost=22.95..2583.12 rows=4032 width=101) (actual rows=4016.00 loops=1) + -> Hash Join (cost=22.95..2758.20 rows=4032 width=101) (actual rows=4016.00 loops=1) Hash Cond: ((f.id1a = d1.a) AND (f.id1b = d1.b)) - -> Seq Scan on bloom_star_multi_fact f (cost=0.00..2539.00 rows=4032 width=89) (actual rows=4016.00 loops=1) + -> Seq Scan on bloom_star_multi_fact f (cost=0.00..2714.08 rows=4032 width=89) (actual rows=4016.00 loops=1) Bloom Filter 1: keys=(id1a, id1b) expected=29.7% checked=99991 rejected=70413 (70.4%) Bloom Filter 2: keys=(id2a, id2b) expected=34.9% checked=29587 rejected=19372 (65.5%) Bloom Filter 3: keys=(id3a, id3b) expected=38.9% checked=10215 rejected=6199 (60.7%) @@ -672,21 +672,21 @@ JOIN bloom_star_multi_dim_7 d7 ON (f.id7a = d7.a AND f.id7b = d7.b) WHERE d1.r < 0.3 AND d2.r < 0.35 AND d3.r < 0.4 AND d4.r < 0.45 AND d5.r < 0.5 AND d6.r < 0.55 AND d7.r < 0.6; QUERY PLAN ---------------------------------------------------------------------------------------------------------------------------------------------------------- - Hash Join (cost=176.01..2725.42 rows=283 width=173) (actual rows=292.00 loops=1) + Hash Join (cost=176.01..2909.59 rows=283 width=173) (actual rows=292.00 loops=1) Hash Cond: ((f.id7a = d7.a) AND (f.id7b = d7.b)) - -> Hash Join (cost=149.07..2696.98 rows=283 width=161) (actual rows=292.00 loops=1) + -> Hash Join (cost=149.07..2881.16 rows=283 width=161) (actual rows=292.00 loops=1) Hash Cond: ((f.id6a = d6.a) AND (f.id6b = d6.b)) - -> Hash Join (cost=122.51..2668.94 rows=283 width=149) (actual rows=292.00 loops=1) + -> Hash Join (cost=122.51..2853.12 rows=283 width=149) (actual rows=292.00 loops=1) Hash Cond: ((f.id5a = d5.a) AND (f.id5b = d5.b)) - -> Hash Join (cost=96.24..2641.19 rows=283 width=137) (actual rows=296.00 loops=1) + -> Hash Join (cost=96.24..2825.37 rows=283 width=137) (actual rows=296.00 loops=1) Hash Cond: ((f.id4a = d4.a) AND (f.id4b = d4.b)) - -> Hash Join (cost=71.02..2614.48 rows=283 width=125) (actual rows=296.00 loops=1) + -> Hash Join (cost=71.02..2798.66 rows=283 width=125) (actual rows=296.00 loops=1) Hash Cond: ((f.id3a = d3.a) AND (f.id3b = d3.b)) - -> Hash Join (cost=46.69..2588.66 rows=283 width=113) (actual rows=298.00 loops=1) + -> Hash Join (cost=46.69..2772.84 rows=283 width=113) (actual rows=298.00 loops=1) Hash Cond: ((f.id2a = d2.a) AND (f.id2b = d2.b)) - -> Hash Join (cost=22.95..2563.44 rows=283 width=101) (actual rows=299.00 loops=1) + -> Hash Join (cost=22.95..2747.62 rows=283 width=101) (actual rows=299.00 loops=1) Hash Cond: ((f.id1a = d1.a) AND (f.id1b = d1.b)) - -> Seq Scan on bloom_star_multi_fact f (cost=0.00..2539.00 rows=283 width=89) (actual rows=299.00 loops=1) + -> Seq Scan on bloom_star_multi_fact f (cost=0.00..2723.18 rows=283 width=89) (actual rows=299.00 loops=1) Bloom Filter 1: keys=(id1a, id1b) expected=29.7% checked=99991 rejected=70413 (70.4%) Bloom Filter 2: keys=(id2a, id2b) expected=34.9% checked=29587 rejected=19372 (65.5%) Bloom Filter 3: keys=(id3a, id3b) expected=38.9% checked=10215 rejected=6199 (60.7%) diff --git a/src/test/regress/expected/join.out b/src/test/regress/expected/join.out index e6dc78f6117..f296952314b 100644 --- a/src/test/regress/expected/join.out +++ b/src/test/regress/expected/join.out @@ -10247,14 +10247,12 @@ select * from fkest f1 Hash Cond: ((f2.x = f1.x) AND (f2.x10b = f1.x10)) -> Seq Scan on fkest f2 Filter: (x100 = 2) - Bloom Filter 1: keys=(x, x10b) -> Hash - Bloom Filter 1 -> Seq Scan on fkest f1 Filter: (x100 = 2) -> Index Scan using fkest_x_x10_x100_idx on fkest f3 Index Cond: (x = f1.x) -(12 rows) +(10 rows) alter table fkest add constraint fk foreign key (x, x10b, x100) references fkest (x, x10, x100); @@ -10466,52 +10464,44 @@ select * from j1 natural join j2; explain (verbose, costs off) select * from j1 inner join (select distinct id from j3) j3 on j1.id = j3.id; - QUERY PLAN ------------------------------------------------------ - Hash Join + QUERY PLAN +----------------------------------------- + Nested Loop Output: j1.id, j3.id Inner Unique: true - Hash Cond: (j1.id = j3.id) - -> Seq Scan on public.j1 - Output: j1.id - Bloom Filter 1: keys=(j1.id) expected=33.3% - -> Hash + Join Filter: (j1.id = j3.id) + -> Unique Output: j3.id - Bloom Filter 1 - -> Unique + -> Sort Output: j3.id - -> Sort + Sort Key: j3.id + -> Seq Scan on public.j3 Output: j3.id - Sort Key: j3.id - -> Seq Scan on public.j3 - Output: j3.id -(17 rows) + -> Seq Scan on public.j1 + Output: j1.id +(13 rows) -- ensure group by clause allows the inner to become unique explain (verbose, costs off) select * from j1 inner join (select id from j3 group by id) j3 on j1.id = j3.id; - QUERY PLAN ------------------------------------------------------ - Hash Join + QUERY PLAN +----------------------------------------- + Nested Loop Output: j1.id, j3.id Inner Unique: true - Hash Cond: (j1.id = j3.id) - -> Seq Scan on public.j1 - Output: j1.id - Bloom Filter 1: keys=(j1.id) expected=33.3% - -> Hash + Join Filter: (j1.id = j3.id) + -> Group Output: j3.id - Bloom Filter 1 - -> Group + Group Key: j3.id + -> Sort Output: j3.id - Group Key: j3.id - -> Sort + Sort Key: j3.id + -> Seq Scan on public.j3 Output: j3.id - Sort Key: j3.id - -> Seq Scan on public.j3 - Output: j3.id -(18 rows) + -> Seq Scan on public.j1 + Output: j1.id +(14 rows) drop table j1; drop table j2; diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 7fff6a720aa..b52528870ef 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -614,16 +614,14 @@ CREATE FUNCTION my_gen_series(int, int) RETURNS SETOF integer SUPPORT test_support_func; EXPLAIN (COSTS OFF) SELECT * FROM tenk1 a JOIN my_gen_series(1,1000) g ON a.unique1 = g; - QUERY PLAN ----------------------------------------------- + QUERY PLAN +---------------------------------------- Hash Join - Hash Cond: (a.unique1 = g.g) - -> Seq Scan on tenk1 a - Bloom Filter 1: keys=(unique1) + Hash Cond: (g.g = a.unique1) + -> Function Scan on my_gen_series g -> Hash - Bloom Filter 1 - -> Function Scan on my_gen_series g -(7 rows) + -> Seq Scan on tenk1 a +(5 rows) EXPLAIN (COSTS OFF) SELECT * FROM tenk1 a JOIN my_gen_series(1,10) g ON a.unique1 = g; diff --git a/src/test/regress/expected/select_parallel.out b/src/test/regress/expected/select_parallel.out index 9d6d213d3e1..933921d1860 100644 --- a/src/test/regress/expected/select_parallel.out +++ b/src/test/regress/expected/select_parallel.out @@ -1125,27 +1125,28 @@ reset role; explain (costs off, verbose) select count(*) from tenk1 a where (unique1, two) in (select unique1, row_number() over() from tenk1 b); - QUERY PLAN ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +---------------------------------------------------------------------------------------- Aggregate Output: count(*) - -> Hash Semi Join - Hash Cond: ((a.unique1 = b.unique1) AND (a.two = (row_number() OVER w1))) - -> Seq Scan on public.tenk1 a - Output: a.unique1, a.unique2, a.two, a.four, a.ten, a.twenty, a.hundred, a.thousand, a.twothousand, a.fivethous, a.tenthous, a.odd, a.even, a.stringu1, a.stringu2, a.string4 - Bloom Filter 1: keys=(a.unique1, a.two) expected=0.5% - -> Hash - Output: b.unique1, (row_number() OVER w1) - Bloom Filter 1 - -> WindowAgg - Output: b.unique1, row_number() OVER w1 - Window: w1 AS (ROWS UNBOUNDED PRECEDING) - -> Gather + -> Hash Right Semi Join + Hash Cond: ((b.unique1 = a.unique1) AND ((row_number() OVER w1) = a.two)) + -> WindowAgg + Output: b.unique1, row_number() OVER w1 + Window: w1 AS (ROWS UNBOUNDED PRECEDING) + -> Gather + Output: b.unique1 + Workers Planned: 4 + -> Parallel Index Only Scan using tenk1_unique1 on public.tenk1 b Output: b.unique1 - Workers Planned: 4 - -> Parallel Index Only Scan using tenk1_unique1 on public.tenk1 b - Output: b.unique1 -(18 rows) + -> Hash + Output: a.unique1, a.two + -> Gather + Output: a.unique1, a.two + Workers Planned: 4 + -> Parallel Seq Scan on public.tenk1 a + Output: a.unique1, a.two +(19 rows) -- LIMIT/OFFSET within sub-selects can't be pushed to workers. explain (costs off) -- 2.50.1 (Apple Git-155)