From 936d12f1bd56bfdbd665ae2de659158938ebffc4 Mon Sep 17 00:00:00 2001 From: William Bernbaum Date: Wed, 26 Aug 2026 16:46:50 -0700 Subject: [PATCH v1 13/16] Deduplicate under idempotent aggregates MIN and MAX return the same result however often a row arrives, as does any aggregate carrying DISTINCT, so the query's grouping keys can drive a deduplication below the joins with the aggregates computed above. The aggregates must only read relations the query groups by. --- src/backend/optimizer/plan/initsplan.c | 118 +++++++++++++ src/backend/optimizer/plan/planagg.c | 4 +- src/include/optimizer/planmain.h | 1 + src/test/regress/expected/eager_aggregate.out | 163 +++++++++++++++++- src/test/regress/sql/eager_aggregate.sql | 83 +++++++++ 5 files changed, 357 insertions(+), 12 deletions(-) diff --git a/src/backend/optimizer/plan/initsplan.c b/src/backend/optimizer/plan/initsplan.c index afec4939654..556daaad1a5 100644 --- a/src/backend/optimizer/plan/initsplan.c +++ b/src/backend/optimizer/plan/initsplan.c @@ -100,12 +100,15 @@ static void create_agg_clause_infos(PlannerInfo *root); static bool grouping_key_usable(Expr *expr); static Index max_sortgroupref(List *tlist); static List *pull_agg_level_exprs(PlannerInfo *root); +static Relids grouping_key_relids(PlannerInfo *root); static bool aggref_is_plain(Aggref *aggref); +static bool aggref_idempotent(Aggref *aggref); static bool collect_distinct_agg_keys(Aggref *aggref, Index *nextref, List **exprs, List **clauses); static void add_grouping_expr_infos(PlannerInfo *root, List *exprs, List *clauses); static bool create_distinct_agg_grouping(PlannerInfo *root); +static bool aggs_indifferent_to_duplicates(PlannerInfo *root); static void create_grouping_expr_infos(PlannerInfo *root); static EquivalenceClass *get_eclass_for_sortgroupclause(PlannerInfo *root, SortGroupClause *sgc, @@ -754,6 +757,20 @@ setup_eager_aggregation(PlannerInfo *root) return; } + /* + * An idempotent aggregate ignores how often a row arrives, so a + * deduplication on the query's grouping keys takes the place of a partial + * aggregate. The checks below govern aggregates, so return ahead of + * them. + */ + if (aggs_indifferent_to_duplicates(root)) + { + root->eager_dedup_only = true; + create_grouping_expr_infos(root); + root->filter_only_rels = find_filter_only_rels(root); + return; + } + /* * For now we don't try to support DISTINCT or ORDER BY aggregates. */ @@ -1068,6 +1085,27 @@ pull_agg_level_exprs(PlannerInfo *root) return exprs; } +/* + * grouping_key_relids + * The relations the query groups by. + */ +static Relids +grouping_key_relids(PlannerInfo *root) +{ + Relids relids = NULL; + ListCell *lc; + + foreach(lc, root->eager_group_clause) + { + SortGroupClause *sgc = lfirst_node(SortGroupClause, lc); + Node *expr = get_sortgroupclause_expr(sgc, root->processed_tlist); + + relids = bms_add_members(relids, pull_varnos(root, expr)); + } + + return relids; +} + /* * aggref_is_plain * Is the aggregate free of the decorations we cannot see through? @@ -1086,6 +1124,26 @@ aggref_is_plain(Aggref *aggref) !aggref->aggvariadic); } +/* + * aggref_idempotent + * Does the aggregate return the same result however often a row arrives? + * + * DISTINCT discards the duplicates before aggregating, so any aggregate + * carrying it qualifies. So do MIN and MAX, which the catalog marks by giving + * them a sort operator. + */ +static bool +aggref_idempotent(Aggref *aggref) +{ + if (!aggref_is_plain(aggref)) + return false; + + if (aggref->aggdistinct != NIL) + return true; + + return OidIsValid(fetch_agg_sort_op(aggref->aggfnoid)); +} + /* * collect_distinct_agg_keys * Make a grouping key of each expression the aggregate takes DISTINCT. @@ -1225,6 +1283,66 @@ create_distinct_agg_grouping(PlannerInfo *root) return true; } +/* + * aggs_indifferent_to_duplicates + * Do the query's aggregates return the same results however often a row + * arrives? + * + * A deduplication leaves each surviving row a single time, so every aggregate + * computed above it has to be idempotent. + * + * The aggregates must only read relations the query groups by. A partial + * aggregate on such a relation groups on the same keys. + */ +static bool +aggs_indifferent_to_duplicates(PlannerInfo *root) +{ + List *exprs; + Relids group_relids; + bool result = true; + ListCell *lc; + + if (!root->parse->hasAggs) + return false; + + group_relids = grouping_key_relids(root); + exprs = pull_agg_level_exprs(root); + + foreach(lc, exprs) + { + Expr *expr = (Expr *) lfirst(lc); + Aggref *aggref; + Relids arg_relids; + + /* + * A plain Var at this level is a grouping key, or a grouped primary + * key of the same relation. + */ + if (!IsA(expr, Aggref)) + continue; + + aggref = (Aggref *) expr; + + if (!aggref_idempotent(aggref)) + { + result = false; + break; + } + + arg_relids = pull_varnos(root, (Node *) aggref->args); + result = bms_is_subset(arg_relids, group_relids); + bms_free(arg_relids); + + if (!result) + break; + } + + list_free(exprs); + bms_free(group_relids); + + return result; +} + /* * create_grouping_expr_infos * Create a GroupingExprInfo for each expression usable as grouping key. diff --git a/src/backend/optimizer/plan/planagg.c b/src/backend/optimizer/plan/planagg.c index 75f6475cb56..383c9177adc 100644 --- a/src/backend/optimizer/plan/planagg.c +++ b/src/backend/optimizer/plan/planagg.c @@ -52,8 +52,6 @@ static bool build_minmax_path(PlannerInfo *root, MinMaxAggInfo *mminfo, Oid eqop, Oid sortop, bool reverse_sort, bool nulls_first); static void minmax_qp_callback(PlannerInfo *root, void *extra); -static Oid fetch_agg_sort_op(Oid aggfnoid); - /* * preprocess_minmax_aggregates - preprocess MIN/MAX aggregates @@ -499,7 +497,7 @@ minmax_qp_callback(PlannerInfo *root, void *extra) * Get the OID of the sort operator, if any, associated with an aggregate. * Returns InvalidOid if there is no such operator. */ -static Oid +Oid fetch_agg_sort_op(Oid aggfnoid) { HeapTuple aggTuple; diff --git a/src/include/optimizer/planmain.h b/src/include/optimizer/planmain.h index 71c043a25e8..ddc9aca7de1 100644 --- a/src/include/optimizer/planmain.h +++ b/src/include/optimizer/planmain.h @@ -35,6 +35,7 @@ extern RelOptInfo *query_planner(PlannerInfo *root, * prototypes for plan/planagg.c */ extern void preprocess_minmax_aggregates(PlannerInfo *root); +extern Oid fetch_agg_sort_op(Oid aggfnoid); /* * prototypes for plan/createplan.c diff --git a/src/test/regress/expected/eager_aggregate.out b/src/test/regress/expected/eager_aggregate.out index 12302ee14cb..64929d07d68 100644 --- a/src/test/regress/expected/eager_aggregate.out +++ b/src/test/regress/expected/eager_aggregate.out @@ -2925,19 +2925,23 @@ SELECT a1.id, count(DISTINCT a1.title) JOIN eager_distinct_a2 a2 ON a2.a1_id = a1.id WHERE a2.flag GROUP BY a1.id; - QUERY PLAN ----------------------------------------------------------- + QUERY PLAN +---------------------------------------------------------------- GroupAggregate Group Key: a1.id -> Sort Sort Key: a1.id, a1.title - -> Hash Join - Hash Cond: (a2.a1_id = a1.id) - -> Seq Scan on eager_distinct_a2 a2 - Filter: flag - -> Hash - -> Seq Scan on eager_distinct_a1 a1 -(10 rows) + -> HashAggregate + Group Key: a1.id + -> Hash Join + Hash Cond: (a2.a1_id = a1.id) + -> Partial HashAggregate + Group Key: a2.a1_id + -> Seq Scan on eager_distinct_a2 a2 + Filter: flag + -> Hash + -> Seq Scan on eager_distinct_a1 a1 +(14 rows) -- The deduplication threshold governs the derived keys too SET min_eager_distinct_group_size TO 8; @@ -3053,3 +3057,144 @@ RESET max_parallel_workers_per_gather; DROP TABLE eager_distinct_a1; DROP TABLE eager_distinct_a2; DROP TABLE eager_distinct_a3; +-- +-- Test that an aggregate whose result depends on how many times a row reached +-- it keeps that row observable, so the relation producing it is not reduced to +-- an existence check +-- +CREATE TABLE eager_minmax_d (id int PRIMARY KEY, k text); +CREATE TABLE eager_minmax_f1 (id int PRIMARY KEY, d_id int); +CREATE TABLE eager_minmax_f2 (id int PRIMARY KEY, f1_id int, flag bool); +INSERT INTO eager_minmax_d SELECT i, 'd' || i FROM generate_series(1, 100) i; +INSERT INTO eager_minmax_f1 + SELECT i, ((i - 1) / 10) + 1 FROM generate_series(1, 1000) i; +INSERT INTO eager_minmax_f2 + SELECT i, ((i - 1) / 10) + 1, i % 2 = 0 FROM generate_series(1, 10000) i; +CREATE INDEX ON eager_minmax_f1 (d_id); +CREATE INDEX ON eager_minmax_f2 (f1_id); +ANALYZE eager_minmax_d; +ANALYZE eager_minmax_f1; +ANALYZE eager_minmax_f2; +-- An aggregate reading the other side needs its rows, so that side is +-- partially aggregated instead of being folded away +EXPLAIN (COSTS OFF) +SELECT d.id, max(f1.id) + FROM eager_minmax_d d + JOIN eager_minmax_f1 f1 ON f1.d_id = d.id + GROUP BY d.id; + QUERY PLAN +-------------------------------------------------- + Finalize HashAggregate + Group Key: d.id + -> Hash Join + Hash Cond: (f1.d_id = d.id) + -> Partial HashAggregate + Group Key: f1.d_id + -> Seq Scan on eager_minmax_f1 f1 + -> Hash + -> Seq Scan on eager_minmax_d d +(9 rows) + +-- Counting the matches still requires every one of them +EXPLAIN (COSTS OFF) +SELECT d.id, count(*) + FROM eager_minmax_d d + JOIN eager_minmax_f1 f1 ON f1.d_id = d.id + JOIN eager_minmax_f2 f2 ON f2.f1_id = f1.id + WHERE f2.flag + GROUP BY d.id; + QUERY PLAN +-------------------------------------------------------------- + Finalize HashAggregate + Group Key: d.id + -> Hash Join + Hash Cond: (f1.d_id = d.id) + -> Partial HashAggregate + Group Key: f1.d_id + -> Hash Join + Hash Cond: (f2.f1_id = f1.id) + -> Seq Scan on eager_minmax_f2 f2 + Filter: flag + -> Hash + -> Seq Scan on eager_minmax_f1 f1 + -> Hash + -> Seq Scan on eager_minmax_d d +(14 rows) + +-- ... and so does an aggregate that adds them up +EXPLAIN (COSTS OFF) +SELECT d.id, sum(d.id) + FROM eager_minmax_d d + JOIN eager_minmax_f1 f1 ON f1.d_id = d.id + JOIN eager_minmax_f2 f2 ON f2.f1_id = f1.id + WHERE f2.flag + GROUP BY d.id; + QUERY PLAN +-------------------------------------------------------- + HashAggregate + Group Key: d.id + -> Hash Join + Hash Cond: (f1.d_id = d.id) + -> Hash Join + Hash Cond: (f2.f1_id = f1.id) + -> Seq Scan on eager_minmax_f2 f2 + Filter: flag + -> Hash + -> Seq Scan on eager_minmax_f1 f1 + -> Hash + -> Seq Scan on eager_minmax_d d +(12 rows) + +-- bit_xor is not idempotent, since a second copy of a row cancels the first +EXPLAIN (COSTS OFF) +SELECT d.id, bit_xor(d.id) + FROM eager_minmax_d d + JOIN eager_minmax_f1 f1 ON f1.d_id = d.id + JOIN eager_minmax_f2 f2 ON f2.f1_id = f1.id + WHERE f2.flag + GROUP BY d.id; + QUERY PLAN +-------------------------------------------------------- + HashAggregate + Group Key: d.id + -> Hash Join + Hash Cond: (f1.d_id = d.id) + -> Hash Join + Hash Cond: (f2.f1_id = f1.id) + -> Seq Scan on eager_minmax_f2 f2 + Filter: flag + -> Hash + -> Seq Scan on eager_minmax_f1 f1 + -> Hash + -> Seq Scan on eager_minmax_d d +(12 rows) + +SELECT count(*), min(m), max(m) FROM ( + SELECT d.id, max(d.k) AS m + FROM eager_minmax_d d + JOIN eager_minmax_f1 f1 ON f1.d_id = d.id + JOIN eager_minmax_f2 f2 ON f2.f1_id = f1.id + WHERE f2.flag + GROUP BY d.id) s; + count | min | max +-------+-----+----- + 100 | d1 | d99 +(1 row) + +SET enable_eager_aggregate TO off; +SELECT count(*), min(m), max(m) FROM ( + SELECT d.id, max(d.k) AS m + FROM eager_minmax_d d + JOIN eager_minmax_f1 f1 ON f1.d_id = d.id + JOIN eager_minmax_f2 f2 ON f2.f1_id = f1.id + WHERE f2.flag + GROUP BY d.id) s; + count | min | max +-------+-----+----- + 100 | d1 | d99 +(1 row) + +RESET enable_eager_aggregate; +DROP TABLE eager_minmax_d; +DROP TABLE eager_minmax_f1; +DROP TABLE eager_minmax_f2; diff --git a/src/test/regress/sql/eager_aggregate.sql b/src/test/regress/sql/eager_aggregate.sql index 224d2b98123..cd37db8e79a 100644 --- a/src/test/regress/sql/eager_aggregate.sql +++ b/src/test/regress/sql/eager_aggregate.sql @@ -977,3 +977,86 @@ RESET max_parallel_workers_per_gather; DROP TABLE eager_distinct_a1; DROP TABLE eager_distinct_a2; DROP TABLE eager_distinct_a3; + + +-- +-- Test that an aggregate whose result depends on how many times a row reached +-- it keeps that row observable, so the relation producing it is not reduced to +-- an existence check +-- + +CREATE TABLE eager_minmax_d (id int PRIMARY KEY, k text); +CREATE TABLE eager_minmax_f1 (id int PRIMARY KEY, d_id int); +CREATE TABLE eager_minmax_f2 (id int PRIMARY KEY, f1_id int, flag bool); + +INSERT INTO eager_minmax_d SELECT i, 'd' || i FROM generate_series(1, 100) i; +INSERT INTO eager_minmax_f1 + SELECT i, ((i - 1) / 10) + 1 FROM generate_series(1, 1000) i; +INSERT INTO eager_minmax_f2 + SELECT i, ((i - 1) / 10) + 1, i % 2 = 0 FROM generate_series(1, 10000) i; + +CREATE INDEX ON eager_minmax_f1 (d_id); +CREATE INDEX ON eager_minmax_f2 (f1_id); + +ANALYZE eager_minmax_d; +ANALYZE eager_minmax_f1; +ANALYZE eager_minmax_f2; + +-- An aggregate reading the other side needs its rows, so that side is +-- partially aggregated instead of being folded away +EXPLAIN (COSTS OFF) +SELECT d.id, max(f1.id) + FROM eager_minmax_d d + JOIN eager_minmax_f1 f1 ON f1.d_id = d.id + GROUP BY d.id; + +-- Counting the matches still requires every one of them +EXPLAIN (COSTS OFF) +SELECT d.id, count(*) + FROM eager_minmax_d d + JOIN eager_minmax_f1 f1 ON f1.d_id = d.id + JOIN eager_minmax_f2 f2 ON f2.f1_id = f1.id + WHERE f2.flag + GROUP BY d.id; + +-- ... and so does an aggregate that adds them up +EXPLAIN (COSTS OFF) +SELECT d.id, sum(d.id) + FROM eager_minmax_d d + JOIN eager_minmax_f1 f1 ON f1.d_id = d.id + JOIN eager_minmax_f2 f2 ON f2.f1_id = f1.id + WHERE f2.flag + GROUP BY d.id; + +-- bit_xor is not idempotent, since a second copy of a row cancels the first +EXPLAIN (COSTS OFF) +SELECT d.id, bit_xor(d.id) + FROM eager_minmax_d d + JOIN eager_minmax_f1 f1 ON f1.d_id = d.id + JOIN eager_minmax_f2 f2 ON f2.f1_id = f1.id + WHERE f2.flag + GROUP BY d.id; + +SELECT count(*), min(m), max(m) FROM ( + SELECT d.id, max(d.k) AS m + FROM eager_minmax_d d + JOIN eager_minmax_f1 f1 ON f1.d_id = d.id + JOIN eager_minmax_f2 f2 ON f2.f1_id = f1.id + WHERE f2.flag + GROUP BY d.id) s; + +SET enable_eager_aggregate TO off; + +SELECT count(*), min(m), max(m) FROM ( + SELECT d.id, max(d.k) AS m + FROM eager_minmax_d d + JOIN eager_minmax_f1 f1 ON f1.d_id = d.id + JOIN eager_minmax_f2 f2 ON f2.f1_id = f1.id + WHERE f2.flag + GROUP BY d.id) s; + +RESET enable_eager_aggregate; + +DROP TABLE eager_minmax_d; +DROP TABLE eager_minmax_f1; +DROP TABLE eager_minmax_f2;