From 1388eee8075d776a519fe1d5a204992bdb6f9baa Mon Sep 17 00:00:00 2001 From: William Bernbaum Date: Wed, 26 Aug 2026 16:46:50 -0700 Subject: [PATCH v2 14/17] Deduplicate under idempotent aggregates MIN and MAX return the same result however often a row arrives, as does any aggregate carrying DISTINCT, so the query's grouping keys can drive a deduplication below the joins with the aggregates computed above. The aggregates must only read relations the query groups by. --- src/backend/optimizer/plan/initsplan.c | 116 ++++++++++++ src/backend/optimizer/plan/planagg.c | 4 +- src/include/optimizer/planmain.h | 1 + src/test/regress/expected/eager_aggregate.out | 168 ++++++++++++++++-- src/test/regress/sql/eager_aggregate.sql | 88 ++++++++- 5 files changed, 359 insertions(+), 18 deletions(-) diff --git a/src/backend/optimizer/plan/initsplan.c b/src/backend/optimizer/plan/initsplan.c index b80b8ede289..417dbc0d9b7 100644 --- a/src/backend/optimizer/plan/initsplan.c +++ b/src/backend/optimizer/plan/initsplan.c @@ -101,12 +101,15 @@ static void create_agg_clause_infos(PlannerInfo *root); static bool grouping_key_usable(Expr *expr); static Index max_sortgroupref(List *tlist); static List *pull_agg_level_exprs(PlannerInfo *root); +static Relids grouping_key_relids(PlannerInfo *root); static bool aggref_is_plain(Aggref *aggref); +static bool aggref_idempotent(Aggref *aggref); static bool collect_distinct_agg_keys(Aggref *aggref, Index *nextref, List **exprs, List **clauses); static void add_grouping_expr_infos(PlannerInfo *root, List *exprs, List *clauses); static void create_distinct_agg_grouping(PlannerInfo *root); +static bool aggs_indifferent_to_duplicates(PlannerInfo *root); static void create_grouping_expr_infos(PlannerInfo *root); static EquivalenceClass *get_eclass_for_sortgroupclause(PlannerInfo *root, SortGroupClause *sgc, @@ -718,6 +721,18 @@ collect_eager_agg_infos(PlannerInfo *root) return; } + /* + * An idempotent aggregate ignores how often a row arrives, so a + * deduplication on the query's grouping keys takes the place of a partial + * aggregate. The checks below govern aggregates, so return ahead of + * them. + */ + if (aggs_indifferent_to_duplicates(root)) + { + create_grouping_expr_infos(root); + return; + } + /* * For now we don't try to support DISTINCT or ORDER BY aggregates. */ @@ -1030,6 +1045,27 @@ pull_agg_level_exprs(PlannerInfo *root) return exprs; } +/* + * grouping_key_relids + * The relations the query groups by. + */ +static Relids +grouping_key_relids(PlannerInfo *root) +{ + Relids relids = NULL; + ListCell *lc; + + foreach(lc, root->eager_group_clause) + { + SortGroupClause *sgc = lfirst_node(SortGroupClause, lc); + Node *expr = get_sortgroupclause_expr(sgc, root->processed_tlist); + + relids = bms_add_members(relids, pull_varnos(root, expr)); + } + + return relids; +} + /* * aggref_is_plain * Does the aggregate depend only on the set of its argument values? @@ -1048,6 +1084,26 @@ aggref_is_plain(Aggref *aggref) !aggref->aggvariadic); } +/* + * aggref_idempotent + * Does the aggregate return the same result however often a row arrives? + * + * DISTINCT discards the duplicates before aggregating, so any aggregate + * carrying it qualifies. So do MIN and MAX, which the catalog marks by giving + * them a sort operator. + */ +static bool +aggref_idempotent(Aggref *aggref) +{ + if (!aggref_is_plain(aggref)) + return false; + + if (aggref->aggdistinct != NIL) + return true; + + return OidIsValid(fetch_agg_sort_op(aggref->aggfnoid)); +} + /* * collect_distinct_agg_keys * Make a grouping key of each expression the aggregate takes DISTINCT. @@ -1185,6 +1241,66 @@ create_distinct_agg_grouping(PlannerInfo *root) root->eager_group_clause = clauses; } +/* + * aggs_indifferent_to_duplicates + * Do the query's aggregates return the same results however often a row + * arrives? + * + * A deduplication leaves each surviving row a single time, so every aggregate + * computed above it has to be idempotent. + * + * The aggregates must only read relations the query groups by. A partial + * aggregate on such a relation groups on the same keys. + */ +static bool +aggs_indifferent_to_duplicates(PlannerInfo *root) +{ + List *exprs; + Relids group_relids; + bool result = true; + ListCell *lc; + + if (!root->parse->hasAggs) + return false; + + group_relids = grouping_key_relids(root); + exprs = pull_agg_level_exprs(root); + + foreach(lc, exprs) + { + Expr *expr = (Expr *) lfirst(lc); + Aggref *aggref; + Relids arg_relids; + + /* + * A plain Var at this level is a grouping key, or a grouped primary + * key of the same relation. + */ + if (!IsA(expr, Aggref)) + continue; + + aggref = (Aggref *) expr; + + if (!aggref_idempotent(aggref)) + { + result = false; + break; + } + + arg_relids = pull_varnos(root, (Node *) aggref->args); + result = bms_is_subset(arg_relids, group_relids); + bms_free(arg_relids); + + if (!result) + break; + } + + list_free(exprs); + bms_free(group_relids); + + return result; +} + /* * create_grouping_expr_infos * Create a GroupingExprInfo for each expression usable as grouping key. diff --git a/src/backend/optimizer/plan/planagg.c b/src/backend/optimizer/plan/planagg.c index 75f6475cb56..383c9177adc 100644 --- a/src/backend/optimizer/plan/planagg.c +++ b/src/backend/optimizer/plan/planagg.c @@ -52,8 +52,6 @@ static bool build_minmax_path(PlannerInfo *root, MinMaxAggInfo *mminfo, Oid eqop, Oid sortop, bool reverse_sort, bool nulls_first); static void minmax_qp_callback(PlannerInfo *root, void *extra); -static Oid fetch_agg_sort_op(Oid aggfnoid); - /* * preprocess_minmax_aggregates - preprocess MIN/MAX aggregates @@ -499,7 +497,7 @@ minmax_qp_callback(PlannerInfo *root, void *extra) * Get the OID of the sort operator, if any, associated with an aggregate. * Returns InvalidOid if there is no such operator. */ -static Oid +Oid fetch_agg_sort_op(Oid aggfnoid) { HeapTuple aggTuple; diff --git a/src/include/optimizer/planmain.h b/src/include/optimizer/planmain.h index c23ab568b98..45195327eac 100644 --- a/src/include/optimizer/planmain.h +++ b/src/include/optimizer/planmain.h @@ -35,6 +35,7 @@ extern RelOptInfo *query_planner(PlannerInfo *root, * prototypes for plan/planagg.c */ extern void preprocess_minmax_aggregates(PlannerInfo *root); +extern Oid fetch_agg_sort_op(Oid aggfnoid); /* * prototypes for plan/createplan.c diff --git a/src/test/regress/expected/eager_aggregate.out b/src/test/regress/expected/eager_aggregate.out index 113e6996a47..3e9451d2a7a 100644 --- a/src/test/regress/expected/eager_aggregate.out +++ b/src/test/regress/expected/eager_aggregate.out @@ -2917,28 +2917,31 @@ HAVING count(*) > 5; -> Seq Scan on eager_distinct_a1 a1 (10 rows) --- Keys are derived only for a query with no grouping clause of its own. --- This one groups by a1.id, so eager aggregation declines its DISTINCT --- aggregate. +-- A DISTINCT aggregate is idempotent, so the query's own grouping clause +-- drives the deduplication EXPLAIN (COSTS OFF) SELECT a1.id, count(DISTINCT a1.title) FROM eager_distinct_a1 a1 JOIN eager_distinct_a2 a2 ON a2.a1_id = a1.id WHERE a2.flag GROUP BY a1.id; - QUERY PLAN ----------------------------------------------------------- + QUERY PLAN +---------------------------------------------------------------- GroupAggregate Group Key: a1.id -> Sort Sort Key: a1.id, a1.title - -> Hash Join - Hash Cond: (a2.a1_id = a1.id) - -> Seq Scan on eager_distinct_a2 a2 - Filter: flag - -> Hash - -> Seq Scan on eager_distinct_a1 a1 -(10 rows) + -> HashAggregate + Group Key: a1.id + -> Hash Join + Hash Cond: (a2.a1_id = a1.id) + -> HashAggregate + Group Key: a2.a1_id + -> Seq Scan on eager_distinct_a2 a2 + Filter: flag + -> Hash + -> Seq Scan on eager_distinct_a1 a1 +(14 rows) -- The deduplication threshold governs the derived keys too SET min_eager_distinct_group_size TO 8; @@ -3054,3 +3057,144 @@ RESET max_parallel_workers_per_gather; DROP TABLE eager_distinct_a1; DROP TABLE eager_distinct_a2; DROP TABLE eager_distinct_a3; +-- +-- Test that an aggregate whose result depends on how many times a row reached +-- it keeps that row observable, so the relation producing it still produces +-- every match +-- +CREATE TABLE eager_minmax_d (id int PRIMARY KEY, k text); +CREATE TABLE eager_minmax_f1 (id int PRIMARY KEY, d_id int); +CREATE TABLE eager_minmax_f2 (id int PRIMARY KEY, f1_id int, flag bool); +INSERT INTO eager_minmax_d SELECT i, 'd' || i FROM generate_series(1, 100) i; +INSERT INTO eager_minmax_f1 + SELECT i, ((i - 1) / 10) + 1 FROM generate_series(1, 1000) i; +INSERT INTO eager_minmax_f2 + SELECT i, ((i - 1) / 10) + 1, i % 2 = 0 FROM generate_series(1, 10000) i; +CREATE INDEX ON eager_minmax_f1 (d_id); +CREATE INDEX ON eager_minmax_f2 (f1_id); +ANALYZE eager_minmax_d; +ANALYZE eager_minmax_f1; +ANALYZE eager_minmax_f2; +-- An aggregate reading the other side needs its rows, so that side is +-- partially aggregated instead of being folded away +EXPLAIN (COSTS OFF) +SELECT d.id, max(f1.id) + FROM eager_minmax_d d + JOIN eager_minmax_f1 f1 ON f1.d_id = d.id + GROUP BY d.id; + QUERY PLAN +-------------------------------------------------- + Finalize HashAggregate + Group Key: d.id + -> Hash Join + Hash Cond: (f1.d_id = d.id) + -> Partial HashAggregate + Group Key: f1.d_id + -> Seq Scan on eager_minmax_f1 f1 + -> Hash + -> Seq Scan on eager_minmax_d d +(9 rows) + +-- Counting the matches still requires the join to produce them +EXPLAIN (COSTS OFF) +SELECT d.id, count(*) + FROM eager_minmax_d d + JOIN eager_minmax_f1 f1 ON f1.d_id = d.id + JOIN eager_minmax_f2 f2 ON f2.f1_id = f1.id + WHERE f2.flag + GROUP BY d.id; + QUERY PLAN +-------------------------------------------------------------- + Finalize HashAggregate + Group Key: d.id + -> Hash Join + Hash Cond: (f1.d_id = d.id) + -> Partial HashAggregate + Group Key: f1.d_id + -> Hash Join + Hash Cond: (f2.f1_id = f1.id) + -> Seq Scan on eager_minmax_f2 f2 + Filter: flag + -> Hash + -> Seq Scan on eager_minmax_f1 f1 + -> Hash + -> Seq Scan on eager_minmax_d d +(14 rows) + +-- ... and so does an aggregate that adds them up +EXPLAIN (COSTS OFF) +SELECT d.id, sum(d.id) + FROM eager_minmax_d d + JOIN eager_minmax_f1 f1 ON f1.d_id = d.id + JOIN eager_minmax_f2 f2 ON f2.f1_id = f1.id + WHERE f2.flag + GROUP BY d.id; + QUERY PLAN +-------------------------------------------------------- + HashAggregate + Group Key: d.id + -> Hash Join + Hash Cond: (f1.d_id = d.id) + -> Hash Join + Hash Cond: (f2.f1_id = f1.id) + -> Seq Scan on eager_minmax_f2 f2 + Filter: flag + -> Hash + -> Seq Scan on eager_minmax_f1 f1 + -> Hash + -> Seq Scan on eager_minmax_d d +(12 rows) + +-- bit_xor is not idempotent, since a second copy of a row cancels the first +EXPLAIN (COSTS OFF) +SELECT d.id, bit_xor(d.id) + FROM eager_minmax_d d + JOIN eager_minmax_f1 f1 ON f1.d_id = d.id + JOIN eager_minmax_f2 f2 ON f2.f1_id = f1.id + WHERE f2.flag + GROUP BY d.id; + QUERY PLAN +-------------------------------------------------------- + HashAggregate + Group Key: d.id + -> Hash Join + Hash Cond: (f1.d_id = d.id) + -> Hash Join + Hash Cond: (f2.f1_id = f1.id) + -> Seq Scan on eager_minmax_f2 f2 + Filter: flag + -> Hash + -> Seq Scan on eager_minmax_f1 f1 + -> Hash + -> Seq Scan on eager_minmax_d d +(12 rows) + +SELECT count(*), min(m), max(m) FROM ( + SELECT d.id, max(d.k) AS m + FROM eager_minmax_d d + JOIN eager_minmax_f1 f1 ON f1.d_id = d.id + JOIN eager_minmax_f2 f2 ON f2.f1_id = f1.id + WHERE f2.flag + GROUP BY d.id) s; + count | min | max +-------+-----+----- + 100 | d1 | d99 +(1 row) + +SET enable_eager_aggregate TO off; +SELECT count(*), min(m), max(m) FROM ( + SELECT d.id, max(d.k) AS m + FROM eager_minmax_d d + JOIN eager_minmax_f1 f1 ON f1.d_id = d.id + JOIN eager_minmax_f2 f2 ON f2.f1_id = f1.id + WHERE f2.flag + GROUP BY d.id) s; + count | min | max +-------+-----+----- + 100 | d1 | d99 +(1 row) + +RESET enable_eager_aggregate; +DROP TABLE eager_minmax_d; +DROP TABLE eager_minmax_f1; +DROP TABLE eager_minmax_f2; diff --git a/src/test/regress/sql/eager_aggregate.sql b/src/test/regress/sql/eager_aggregate.sql index 6610fb8fa52..fafa60edc78 100644 --- a/src/test/regress/sql/eager_aggregate.sql +++ b/src/test/regress/sql/eager_aggregate.sql @@ -908,9 +908,8 @@ SELECT count(DISTINCT a1.id) WHERE a2.flag HAVING count(*) > 5; --- Keys are derived only for a query with no grouping clause of its own. --- This one groups by a1.id, so eager aggregation declines its DISTINCT --- aggregate. +-- A DISTINCT aggregate is idempotent, so the query's own grouping clause +-- drives the deduplication EXPLAIN (COSTS OFF) SELECT a1.id, count(DISTINCT a1.title) FROM eager_distinct_a1 a1 @@ -978,3 +977,86 @@ RESET max_parallel_workers_per_gather; DROP TABLE eager_distinct_a1; DROP TABLE eager_distinct_a2; DROP TABLE eager_distinct_a3; + + +-- +-- Test that an aggregate whose result depends on how many times a row reached +-- it keeps that row observable, so the relation producing it still produces +-- every match +-- + +CREATE TABLE eager_minmax_d (id int PRIMARY KEY, k text); +CREATE TABLE eager_minmax_f1 (id int PRIMARY KEY, d_id int); +CREATE TABLE eager_minmax_f2 (id int PRIMARY KEY, f1_id int, flag bool); + +INSERT INTO eager_minmax_d SELECT i, 'd' || i FROM generate_series(1, 100) i; +INSERT INTO eager_minmax_f1 + SELECT i, ((i - 1) / 10) + 1 FROM generate_series(1, 1000) i; +INSERT INTO eager_minmax_f2 + SELECT i, ((i - 1) / 10) + 1, i % 2 = 0 FROM generate_series(1, 10000) i; + +CREATE INDEX ON eager_minmax_f1 (d_id); +CREATE INDEX ON eager_minmax_f2 (f1_id); + +ANALYZE eager_minmax_d; +ANALYZE eager_minmax_f1; +ANALYZE eager_minmax_f2; + +-- An aggregate reading the other side needs its rows, so that side is +-- partially aggregated instead of being folded away +EXPLAIN (COSTS OFF) +SELECT d.id, max(f1.id) + FROM eager_minmax_d d + JOIN eager_minmax_f1 f1 ON f1.d_id = d.id + GROUP BY d.id; + +-- Counting the matches still requires the join to produce them +EXPLAIN (COSTS OFF) +SELECT d.id, count(*) + FROM eager_minmax_d d + JOIN eager_minmax_f1 f1 ON f1.d_id = d.id + JOIN eager_minmax_f2 f2 ON f2.f1_id = f1.id + WHERE f2.flag + GROUP BY d.id; + +-- ... and so does an aggregate that adds them up +EXPLAIN (COSTS OFF) +SELECT d.id, sum(d.id) + FROM eager_minmax_d d + JOIN eager_minmax_f1 f1 ON f1.d_id = d.id + JOIN eager_minmax_f2 f2 ON f2.f1_id = f1.id + WHERE f2.flag + GROUP BY d.id; + +-- bit_xor is not idempotent, since a second copy of a row cancels the first +EXPLAIN (COSTS OFF) +SELECT d.id, bit_xor(d.id) + FROM eager_minmax_d d + JOIN eager_minmax_f1 f1 ON f1.d_id = d.id + JOIN eager_minmax_f2 f2 ON f2.f1_id = f1.id + WHERE f2.flag + GROUP BY d.id; + +SELECT count(*), min(m), max(m) FROM ( + SELECT d.id, max(d.k) AS m + FROM eager_minmax_d d + JOIN eager_minmax_f1 f1 ON f1.d_id = d.id + JOIN eager_minmax_f2 f2 ON f2.f1_id = f1.id + WHERE f2.flag + GROUP BY d.id) s; + +SET enable_eager_aggregate TO off; + +SELECT count(*), min(m), max(m) FROM ( + SELECT d.id, max(d.k) AS m + FROM eager_minmax_d d + JOIN eager_minmax_f1 f1 ON f1.d_id = d.id + JOIN eager_minmax_f2 f2 ON f2.f1_id = f1.id + WHERE f2.flag + GROUP BY d.id) s; + +RESET enable_eager_aggregate; + +DROP TABLE eager_minmax_d; +DROP TABLE eager_minmax_f1; +DROP TABLE eager_minmax_f2;