From 75a7b047904b791014ba097d7cf97957bf44a2e7 Mon Sep 17 00:00:00 2001 From: Xiangxin Zeng Date: Thu, 15 Oct 2026 23:22:00 +0800 Subject: [PATCH 2/2] Use safe ends of UNION ALL ndistinct estimates The low end used by generic get_variable_numdistinct() callers is not conservative for every consumer of a distinct-value count. GROUP BY, DISTINCT, and Memoize estimates use the count as an output cardinality, so use the high end when all members are estimable. With only a partial low end, keep the estimate at least DEFAULT_NUM_DISTINCT and retain the used-default signal; this prevents Memoize from treating a partial low-end count as evidence of a high cache hit rate. The inner side of an equality semi/anti join uses the count as a numerator, so use the high end there too. Accept a partial low end only when it already covers the outer distinct count and therefore saturates the semi-join ratio; otherwise retain the existing uncertain-value heuristic. Add regressions for complete high-end estimates, an empty member, and a partial low end that saturates a semi join. Signed-off-by: Xiangxin Zeng --- src/backend/utils/adt/selfuncs.c | 46 ++++++++++- src/test/regress/expected/join.out | 38 +++++---- src/test/regress/expected/planner_est.out | 93 +++++++++++++++++++++++ src/test/regress/expected/union.out | 48 ++++++------ src/test/regress/sql/planner_est.sql | 68 +++++++++++++++++ 5 files changed, 248 insertions(+), 45 deletions(-) diff --git a/src/backend/utils/adt/selfuncs.c b/src/backend/utils/adt/selfuncs.c index 69783c82e3..d4a04ce405 100644 --- a/src/backend/utils/adt/selfuncs.c +++ b/src/backend/utils/adt/selfuncs.c @@ -2764,6 +2764,27 @@ eqjoinsel_semi(FmgrInfo *eqproc, Oid collation, { double selec; + /* + * The inner-side distinct count is a numerator. Prefer the high end when + * available, so a disjoint UNION ALL does not look like a small domain. A + * low end from partial information is useful only when it is at least + * nd1: the semi-join ratio is then saturated, so missing members cannot + * reduce it further. Otherwise leave the estimate marked as a default. + */ + if (vardata2->ndistinct_hi > 0.0) + { + nd2 = clamp_row_est(vardata2->ndistinct_hi); + isdefault2 = false; + } + else if (vardata2->ndistinct_lo > 0.0 && + clamp_row_est(vardata2->ndistinct_lo) >= nd1) + { + nd2 = clamp_row_est(vardata2->ndistinct_lo); + isdefault2 = false; + } + else if (vardata2->ndistinct_lo > 0.0) + isdefault2 = true; + /* * We clamp nd2 to be not more than what we estimate the inner relation's * size to be. This is intuitively somewhat reasonable since obviously @@ -3682,6 +3703,28 @@ add_unique_group_var(PlannerInfo *root, List *varinfos, ndistinct = get_variable_numdistinct(vardata, &isdefault); + /* + * A group count is a row count. Prefer the high end over the + * divisor-safe low end returned by get_variable_numdistinct(). + */ + if (vardata->ndistinct_hi > 0.0) + { + ndistinct = vardata->ndistinct_hi; + isdefault = false; + } + else if (vardata->ndistinct_lo > 0.0) + { + /* + * No high end is available because some member lacked an estimate. + * Use the low end as a floor, but never below the historical default. + * + * Keep the default flag set: without a high end, Memoize must not + * treat a low-end count as evidence of a high cache hit rate. + */ + ndistinct = Max(vardata->ndistinct_lo, (double) DEFAULT_NUM_DISTINCT); + isdefault = true; + } + /* * The nullingrels bits within the var could cause the same var to be * counted multiple times if it's marked with different nullingrels. They @@ -6316,8 +6359,7 @@ examine_simple_variable(PlannerInfo *root, Var *var, if (rel->attr_ndistinct_lo == NULL) set_append_rel_ndistinct(root, rel); - if (var->varattno <= 0 || - var->varattno > rel->max_attr) + if (var->varattno <= 0) return; pndx = var->varattno - rel->min_attr; diff --git a/src/test/regress/expected/join.out b/src/test/regress/expected/join.out index db4fcc5a5a..b8e8bab9b3 100644 --- a/src/test/regress/expected/join.out +++ b/src/test/regress/expected/join.out @@ -5081,30 +5081,28 @@ select * from join (select 3 as y) on i is not null ), lateral (select k as kl limit 1); - QUERY PLAN -------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------- Nested Loop - Output: COALESCE(t.i, (1)), ((COALESCE(t.i, (1)))) - -> Limit - Output: 1 - -> Result - Output: 1 + Output: (COALESCE(t.i, (1))), ((COALESCE(t.i, (1)))) -> Nested Loop - Output: t.i, ((COALESCE(t.i, (1)))) - -> Result - Output: t.i, COALESCE(t.i, (1)) - -> Append - -> Index Only Scan using t_pkey on pg_temp.t - Output: t.i - Index Cond: (t.i = (1)) - -> Result - Output: 0 - One-Time Filter: ((1) = 0) + Output: COALESCE(t.i, (1)) -> Limit - Output: ((COALESCE(t.i, (1)))) + Output: 1 -> Result - Output: (COALESCE(t.i, (1))) -(21 rows) + Output: 1 + -> Append + -> Index Only Scan using t_pkey on pg_temp.t + Output: t.i + Index Cond: (t.i = (1)) + -> Result + Output: 0 + One-Time Filter: ((1) = 0) + -> Limit + Output: ((COALESCE(t.i, (1)))) + -> Result + Output: (COALESCE(t.i, (1))) +(19 rows) rollback; -- PHVs containing SubLinks are quite tricky to get right diff --git a/src/test/regress/expected/planner_est.out b/src/test/regress/expected/planner_est.out index 236cb274a7..6e2a9b3204 100644 --- a/src/test/regress/expected/planner_est.out +++ b/src/test/regress/expected/planner_est.out @@ -42,10 +42,103 @@ BEGIN ln := regexp_replace(ln, 'width=\d+', 'width=N'); END IF; + ln := regexp_replace(ln, + 'Buckets: \d+ Batches: \d+ Memory Usage: \d+kB', + 'Buckets: N Batches: N Memory Usage: NkB'); + RETURN NEXT ln; END LOOP; END; $$; +-- Ensure that a semi-join inner side made from two disjoint UNION ALL arms +-- uses the high distinct-value estimate. The low end would estimate only +-- 2000 of the 3000 matching outer rows. +CREATE TEMP TABLE semi_union_outer (id int); +CREATE TEMP TABLE semi_union_arm1 (id int); +CREATE TEMP TABLE semi_union_arm2 (id int); +INSERT INTO semi_union_outer SELECT g FROM generate_series(1000, 3999) g; +INSERT INTO semi_union_arm1 SELECT g FROM generate_series(1000, 2999) g; +INSERT INTO semi_union_arm2 SELECT g FROM generate_series(3000, 3999) g; +ANALYZE semi_union_outer; +ANALYZE semi_union_arm1; +ANALYZE semi_union_arm2; +SELECT explain_mask_costs($$ +SELECT * FROM semi_union_outer WHERE id IN + (SELECT id FROM semi_union_arm1 + UNION ALL + SELECT id FROM semi_union_arm2);$$, +true, true, false, true); + explain_mask_costs +------------------------------------------------------------------------------------------------------- + Hash Right Semi Join (cost=N..N rows=3000 width=N) (actual rows=3000.00 loops=1) + Hash Cond: (semi_union_arm1.id = semi_union_outer.id) + -> Append (cost=N..N rows=3000 width=N) (actual rows=3000.00 loops=1) + -> Seq Scan on semi_union_arm1 (cost=N..N rows=2000 width=N) (actual rows=2000.00 loops=1) + -> Seq Scan on semi_union_arm2 (cost=N..N rows=1000 width=N) (actual rows=1000.00 loops=1) + -> Hash (cost=N..N rows=3000 width=N) (actual rows=3000.00 loops=1) + Buckets: N Batches: N Memory Usage: NkB + -> Seq Scan on semi_union_outer (cost=N..N rows=3000 width=N) (actual rows=3000.00 loops=1) +(8 rows) + +DROP TABLE semi_union_outer, semi_union_arm1, semi_union_arm2; +-- A member proven empty must not contribute its one-row group estimate to +-- the union's high distinct-value estimate. +CREATE TEMP TABLE dummy_union_outer (id int); +CREATE TEMP TABLE dummy_union_live (id int); +CREATE TEMP TABLE dummy_union_pruned (id int); +INSERT INTO dummy_union_outer SELECT g FROM generate_series(1, 2002) g; +INSERT INTO dummy_union_live SELECT g FROM generate_series(1, 2000) g; +INSERT INTO dummy_union_pruned SELECT g FROM generate_series(2002, 4001) g; +ANALYZE dummy_union_outer; +ANALYZE dummy_union_live; +ANALYZE dummy_union_pruned; +SELECT explain_mask_costs($$ +SELECT * FROM dummy_union_outer WHERE id IN + (SELECT id FROM dummy_union_live + UNION ALL + SELECT id FROM dummy_union_pruned WHERE false);$$, +true, true, false, true); + explain_mask_costs +------------------------------------------------------------------------------------------------------- + Hash Semi Join (cost=N..N rows=2000 width=N) (actual rows=2000.00 loops=1) + Hash Cond: (dummy_union_outer.id = dummy_union_live.id) + -> Seq Scan on dummy_union_outer (cost=N..N rows=2002 width=N) (actual rows=2002.00 loops=1) + -> Hash (cost=N..N rows=2000 width=N) (actual rows=2000.00 loops=1) + Buckets: N Batches: N Memory Usage: NkB + -> Seq Scan on dummy_union_live (cost=N..N rows=2000 width=N) (actual rows=2000.00 loops=1) +(6 rows) + +DROP TABLE dummy_union_outer, dummy_union_live, dummy_union_pruned; +-- A partial low-end estimate is usable by a semi join once it already covers +-- the outer side's distinct-value count, even though another member has no +-- statistics. +CREATE TEMP TABLE partial_union_outer (id int); +CREATE TEMP TABLE partial_union_known (id int); +CREATE TEMP TABLE partial_union_unknown (id int); +INSERT INTO partial_union_outer SELECT g FROM generate_series(1, 1000) g; +INSERT INTO partial_union_known SELECT g FROM generate_series(1, 1000) g; +INSERT INTO partial_union_unknown SELECT g FROM generate_series(1001, 2000) g; +ANALYZE partial_union_outer; +ANALYZE partial_union_known; +SELECT explain_mask_costs($$ +SELECT * FROM partial_union_outer WHERE id IN + (SELECT id FROM partial_union_known + UNION ALL + SELECT id FROM partial_union_unknown);$$, +true, true, false, true); + explain_mask_costs +------------------------------------------------------------------------------------------------------------ + Hash Right Semi Join (cost=N..N rows=1000 width=N) (actual rows=1000.00 loops=1) + Hash Cond: (partial_union_known.id = partial_union_outer.id) + -> Append (cost=N..N rows=3550 width=N) (actual rows=2000.00 loops=1) + -> Seq Scan on partial_union_known (cost=N..N rows=1000 width=N) (actual rows=1000.00 loops=1) + -> Seq Scan on partial_union_unknown (cost=N..N rows=2550 width=N) (actual rows=1000.00 loops=1) + -> Hash (cost=N..N rows=1000 width=N) (actual rows=1000.00 loops=1) + Buckets: N Batches: N Memory Usage: NkB + -> Seq Scan on partial_union_outer (cost=N..N rows=1000 width=N) (actual rows=1000.00 loops=1) +(8 rows) + +DROP TABLE partial_union_outer, partial_union_known, partial_union_unknown; -- -- Test the SupportRequestRows support function for generate_series_timestamp() -- diff --git a/src/test/regress/expected/union.out b/src/test/regress/expected/union.out index 6ce430de99..f113c8a790 100644 --- a/src/test/regress/expected/union.out +++ b/src/test/regress/expected/union.out @@ -1506,22 +1506,24 @@ select distinct q1 from union all select distinct * from int8_tbl i82) ss where q2 = q2; - QUERY PLAN ----------------------------------------------------- - HashAggregate - Group Key: unnamed_subquery.q1 - -> Append + QUERY PLAN +---------------------------------------------------------- + Unique + -> Merge Append + Sort Key: unnamed_subquery.q1 -> Subquery Scan on unnamed_subquery - -> HashAggregate - Group Key: i81.q1, i81.q2 - -> Seq Scan on int8_tbl i81 - Filter: (q2 IS NOT NULL) + -> Unique + -> Sort + Sort Key: i81.q1, i81.q2 + -> Seq Scan on int8_tbl i81 + Filter: (q2 IS NOT NULL) -> Subquery Scan on unnamed_subquery_1 - -> HashAggregate - Group Key: i82.q1, i82.q2 - -> Seq Scan on int8_tbl i82 - Filter: (q2 IS NOT NULL) -(13 rows) + -> Unique + -> Sort + Sort Key: i82.q1, i82.q2 + -> Seq Scan on int8_tbl i82 + Filter: (q2 IS NOT NULL) +(15 rows) select distinct q1 from (select distinct * from int8_tbl i81 @@ -1742,18 +1744,18 @@ select count(*) from ud join (select distinct id from (select id from ua1 union all select id from ua2) x) y on ud.id = y.id; - QUERY PLAN ------------------------------------------------ + QUERY PLAN +----------------------------------------- Aggregate -> Hash Join - Hash Cond: (ud.id = ua1.id) - -> Seq Scan on ud + Hash Cond: (ua1.id = ud.id) + -> HashAggregate + Group Key: ua1.id + -> Append + -> Seq Scan on ua1 + -> Seq Scan on ua2 -> Hash - -> HashAggregate - Group Key: ua1.id - -> Append - -> Seq Scan on ua1 - -> Seq Scan on ua2 + -> Seq Scan on ud (10 rows) -- A member with no statistics cannot contribute to the sum, so the union has diff --git a/src/test/regress/sql/planner_est.sql b/src/test/regress/sql/planner_est.sql index 2b696a4e4e..25d75e8c88 100644 --- a/src/test/regress/sql/planner_est.sql +++ b/src/test/regress/sql/planner_est.sql @@ -43,11 +43,79 @@ BEGIN ln := regexp_replace(ln, 'width=\d+', 'width=N'); END IF; + ln := regexp_replace(ln, + 'Buckets: \d+ Batches: \d+ Memory Usage: \d+kB', + 'Buckets: N Batches: N Memory Usage: NkB'); + RETURN NEXT ln; END LOOP; END; $$; +-- Ensure that a semi-join inner side made from two disjoint UNION ALL arms +-- uses the high distinct-value estimate. The low end would estimate only +-- 2000 of the 3000 matching outer rows. +CREATE TEMP TABLE semi_union_outer (id int); +CREATE TEMP TABLE semi_union_arm1 (id int); +CREATE TEMP TABLE semi_union_arm2 (id int); +INSERT INTO semi_union_outer SELECT g FROM generate_series(1000, 3999) g; +INSERT INTO semi_union_arm1 SELECT g FROM generate_series(1000, 2999) g; +INSERT INTO semi_union_arm2 SELECT g FROM generate_series(3000, 3999) g; +ANALYZE semi_union_outer; +ANALYZE semi_union_arm1; +ANALYZE semi_union_arm2; + +SELECT explain_mask_costs($$ +SELECT * FROM semi_union_outer WHERE id IN + (SELECT id FROM semi_union_arm1 + UNION ALL + SELECT id FROM semi_union_arm2);$$, +true, true, false, true); + +DROP TABLE semi_union_outer, semi_union_arm1, semi_union_arm2; + +-- A member proven empty must not contribute its one-row group estimate to +-- the union's high distinct-value estimate. +CREATE TEMP TABLE dummy_union_outer (id int); +CREATE TEMP TABLE dummy_union_live (id int); +CREATE TEMP TABLE dummy_union_pruned (id int); +INSERT INTO dummy_union_outer SELECT g FROM generate_series(1, 2002) g; +INSERT INTO dummy_union_live SELECT g FROM generate_series(1, 2000) g; +INSERT INTO dummy_union_pruned SELECT g FROM generate_series(2002, 4001) g; +ANALYZE dummy_union_outer; +ANALYZE dummy_union_live; +ANALYZE dummy_union_pruned; + +SELECT explain_mask_costs($$ +SELECT * FROM dummy_union_outer WHERE id IN + (SELECT id FROM dummy_union_live + UNION ALL + SELECT id FROM dummy_union_pruned WHERE false);$$, +true, true, false, true); + +DROP TABLE dummy_union_outer, dummy_union_live, dummy_union_pruned; + +-- A partial low-end estimate is usable by a semi join once it already covers +-- the outer side's distinct-value count, even though another member has no +-- statistics. +CREATE TEMP TABLE partial_union_outer (id int); +CREATE TEMP TABLE partial_union_known (id int); +CREATE TEMP TABLE partial_union_unknown (id int); +INSERT INTO partial_union_outer SELECT g FROM generate_series(1, 1000) g; +INSERT INTO partial_union_known SELECT g FROM generate_series(1, 1000) g; +INSERT INTO partial_union_unknown SELECT g FROM generate_series(1001, 2000) g; +ANALYZE partial_union_outer; +ANALYZE partial_union_known; + +SELECT explain_mask_costs($$ +SELECT * FROM partial_union_outer WHERE id IN + (SELECT id FROM partial_union_known + UNION ALL + SELECT id FROM partial_union_unknown);$$, +true, true, false, true); + +DROP TABLE partial_union_outer, partial_union_known, partial_union_unknown; + -- -- Test the SupportRequestRows support function for generate_series_timestamp() -- -- 2.43.0