From 897460fe425d619d79ed41e11567409f8a0971c3 Mon Sep 17 00:00:00 2001 From: Alexandra Wang Date: Mon, 3 Aug 2026 15:59:52 -0700 Subject: [PATCH v9 2/4] Unify extended statistics columns and expressions into a single stxexprs list Remove the stxkeys int2vector from pg_statistic_ext. A statistics object's columns and expressions are now stored together in stxexprs, in user-written order, with plain columns represented as Var nodes. The columns previously lived in a separate stxkeys array, sorted by attnum, which lost that order. This leaves one representation of them instead of two, in the catalog and in the in-memory StatExtEntry and StatisticExtInfo structs alike. The pg_stats_ext view accordingly drops its attnames column and shows all entries in exprs. Where the code previously kept a keys Bitmapset beside the expression list, the covered column attnums and the per-expression index are now derived from the single list on demand. The user-written order is preserved in the pg_stats_ext view, in the CREATE STATISTICS command that pg_dump reproduces, and in the MCV list. MCV dimensions are positional, so the planner maps a clause to its dimension through the list instead of assuming columns come first. The ndistinct and dependencies data identify their attributes by number, so their order within an item does not affect estimation, but pg_dump and pg_upgrade reload that data through the pg_ndistinct and pg_dependencies input functions, which require a canonical order (columns ascending, then expressions descending). Attnum-sorted stxkeys used to produce that order; statext_ndistinct_build and statext_dependencies_build now sort each item explicitly. Because an entry that reduces to a plain column is treated as that column, it can be used to estimate queries on that column, which the separate stxkeys/stxexprs representation could not do. For the same reason, CREATE STATISTICS now rejects one that duplicates a column already listed (an expression that const-folds to it, or a virtual generated column defined on it), the same way a plainly repeated column is rejected; the duplicate check compares entries after expanding and const-folding them, not structurally. This is a prerequisite refactor for the join statistics patch. Suggested-by: Tom Lane Suggested-by: Tomas Vondra Discussion: https://postgr.es/m/711247.1779913876@sss.pgh.pa.us --- doc/src/sgml/catalogs.sgml | 19 +- doc/src/sgml/perform.sgml | 116 ++-- doc/src/sgml/system-views.sgml | 13 +- src/backend/catalog/system_views.sql | 7 +- src/backend/commands/statscmds.c | 202 +++---- src/backend/optimizer/util/plancat.c | 93 +--- src/backend/parser/parse_utilcmd.c | 65 +-- src/backend/statistics/dependencies.c | 61 ++- src/backend/statistics/extended_stats.c | 516 +++++++++++------- src/backend/statistics/extended_stats_funcs.c | 159 +++--- src/backend/statistics/mcv.c | 75 ++- src/backend/statistics/mvdistinct.c | 23 +- src/backend/utils/adt/ruleutils.c | 304 ++++++----- src/backend/utils/adt/selfuncs.c | 23 +- src/bin/psql/describe.c | 20 +- src/include/catalog/pg_proc.dat | 4 +- src/include/catalog/pg_statistic_ext.h | 15 +- src/include/nodes/pathnodes.h | 5 +- .../statistics/extended_stats_internal.h | 7 +- src/include/statistics/statistics.h | 5 + .../regress/expected/create_table_like.out | 9 +- src/test/regress/expected/oidjoins.out | 1 - src/test/regress/expected/rules.out | 5 +- src/test/regress/expected/stats_ext.out | 143 ++++- src/test/regress/expected/stats_import.out | 119 +++- src/test/regress/sql/create_table_like.sql | 9 +- src/test/regress/sql/stats_ext.sql | 77 ++- src/test/regress/sql/stats_import.sql | 102 ++++ 28 files changed, 1320 insertions(+), 877 deletions(-) diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index 6066c4784f4..f747b9528d2 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -8323,18 +8323,6 @@ SCRAM-SHA-256$<iteration count>:&l - - - stxkeys int2vector - (references pg_attribute.attnum) - - - An array of attribute numbers, indicating which table columns are - covered by this statistics object; - for example a value of 1 3 would - mean that the first and the third table columns are covered - - @@ -8373,9 +8361,10 @@ SCRAM-SHA-256$<iteration count>:&l Expression trees (in nodeToString() - representation) for statistics object attributes that are not simple - column references. This is a list with one element per expression. - Null if all statistics object attributes are simple references. + representation) for the columns and expressions the statistics object + is defined on. Both column + references and expressions are stored here, with one element per + column or expression, in the order they were written. diff --git a/doc/src/sgml/perform.sgml b/doc/src/sgml/perform.sgml index 604e8578a8d..ba136e2b501 100644 --- a/doc/src/sgml/perform.sgml +++ b/doc/src/sgml/perform.sgml @@ -1541,30 +1541,32 @@ CREATE STATISTICS stts (dependencies) ON city, zip FROM zipcodes; ANALYZE zipcodes; -SELECT stxkeys AS k, jsonb_pretty(stxddependencies::text::jsonb) AS dep +SELECT jsonb_pretty(stxddependencies::text::jsonb) AS dep FROM pg_statistic_ext JOIN pg_statistic_ext_data ON (oid = stxoid) WHERE stxname = 'stts'; --[ RECORD 1 ]-------------------- -k | 1 5 -dep | [ + - | { + - | "degree": 1.000000,+ - | "attributes": [ + - | 1 + - | ], + - | "dependency": 5 + - | }, + - | { + - | "degree": 0.423130,+ - | "attributes": [ + - | 5 + - | ], + - | "dependency": 1 + - | } + - | ] + dep +----------------------------- + [ + + { + + "degree": 1.000000,+ + "attributes": [ + + 1 + + ], + + "dependency": 5 + + }, + + { + + "degree": 0.423130,+ + "attributes": [ + + 5 + + ], + + "dependency": 1 + + } + + ] (1 row) - Here it can be seen that column 1 (zip code) fully determines column + The dependency data identifies columns by attribute number: in this + table zip is attribute 1 and city is attribute 5. It can then be seen + that column 1 (zip code) fully determines column 5 (city) so the coefficient is 1.0, while city only determines zip code about 42% of the time, meaning that there are many cities (58%) that are represented by more than a single ZIP code. @@ -1647,46 +1649,48 @@ CREATE STATISTICS stts2 (ndistinct) ON city, state, zip FROM zipcodes; ANALYZE zipcodes; -SELECT stxkeys AS k, jsonb_pretty(stxdndistinct::text::jsonb) AS nd +SELECT jsonb_pretty(stxdndistinct::text::jsonb) AS nd FROM pg_statistic_ext JOIN pg_statistic_ext_data on (oid = stxoid) WHERE stxname = 'stts2'; --[ RECORD 1 ]------------------- -k | 1 2 5 -nd | [ + - | { + - | "ndistinct": 33178,+ - | "attributes": [ + - | 1, + - | 2 + - | ] + - | }, + - | { + - | "ndistinct": 33178,+ - | "attributes": [ + - | 1, + - | 5 + - | ] + - | }, + - | { + - | "ndistinct": 27435,+ - | "attributes": [ + - | 2, + - | 5 + - | ] + - | }, + - | { + - | "ndistinct": 33178,+ - | "attributes": [ + - | 1, + - | 2, + - | 5 + - | ] + - | } + - | ] + nd +----------------------------- + [ + + { + + "ndistinct": 33178,+ + "attributes": [ + + 1, + + 2 + + ] + + }, + + { + + "ndistinct": 33178,+ + "attributes": [ + + 1, + + 5 + + ] + + }, + + { + + "ndistinct": 27435,+ + "attributes": [ + + 2, + + 5 + + ] + + }, + + { + + "ndistinct": 33178,+ + "attributes": [ + + 1, + + 2, + + 5 + + ] + + } + + ] (1 row) - This indicates that there are three combinations of columns that - have 33178 distinct values: ZIP code and state; ZIP code and city; + As above, the columns appear by attribute number: zip is 1, state is 2 + and city is 5. This indicates that there are three combinations of + columns that have 33178 distinct values: ZIP code and state; ZIP code + and city; and ZIP code, city and state (the fact that they are all equal is expected given that ZIP code alone is unique in this table). On the other hand, the combination of city and state has only 27435 distinct diff --git a/doc/src/sgml/system-views.sgml b/doc/src/sgml/system-views.sgml index 5ea19d68622..f73a9b1f7b2 100644 --- a/doc/src/sgml/system-views.sgml +++ b/doc/src/sgml/system-views.sgml @@ -4737,22 +4737,13 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx - - - attnames name[] - (references pg_attribute.attname) - - - Names of the columns included in the extended statistics object - - - exprs text[] - Expressions included in the extended statistics object + The columns and expressions the statistics object is + defined on, in the order they were written diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 090281a03dd..599c629e004 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -285,12 +285,7 @@ CREATE VIEW pg_stats_ext WITH (security_barrier) AS s.stxname AS statistics_name, s.oid AS statistics_id, pg_get_userbyid(s.stxowner) AS statistics_owner, - ( SELECT array_agg(a.attname ORDER BY a.attnum) - FROM unnest(s.stxkeys) k - JOIN pg_attribute a - ON (a.attrelid = s.stxrelid AND a.attnum = k) - ) AS attnames, - pg_get_statisticsobjdef_expressions(s.oid) as exprs, + pg_get_statisticsobjdef_columns(s.oid) AS exprs, s.stxkind AS kinds, sd.stxdinherit AS inherited, sd.stxdndistinct AS n_distinct, diff --git a/src/backend/commands/statscmds.c b/src/backend/commands/statscmds.c index ce98ebb35ea..ecddbea32ec 100644 --- a/src/backend/commands/statscmds.c +++ b/src/backend/commands/statscmds.c @@ -31,6 +31,7 @@ #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "optimizer/optimizer.h" +#include "rewrite/rewriteHandler.h" #include "statistics/statistics.h" #include "utils/acl.h" #include "utils/builtins.h" @@ -45,26 +46,12 @@ static char *ChooseExtendedStatisticName(const char *name1, const char *name2, const char *label, Oid namespaceid); static char *ChooseExtendedStatisticNameAddition(List *exprs); - -/* qsort comparator for the attnums in CreateStatistics */ -static int -compare_int16(const void *a, const void *b) -{ - int av = *(const int16 *) a; - int bv = *(const int16 *) b; - - /* this can't overflow if int is wider than int16 */ - return (av - bv); -} - /* * CREATE STATISTICS */ ObjectAddress CreateStatistics(CreateStatsStmt *stmt, bool check_rights) { - int16 attnums[STATS_MAX_DIMENSIONS]; - int nattnums = 0; int numcols; char *namestr; NameData stxname; @@ -74,8 +61,9 @@ CreateStatistics(CreateStatsStmt *stmt, bool check_rights) HeapTuple htup; Datum values[Natts_pg_statistic_ext]; bool nulls[Natts_pg_statistic_ext]; - int2vector *stxkeys; List *stxexprs = NIL; + List *folded_exprs; + char *exprsString; Datum exprsDatum; Relation statrel; Relation rel = NULL; @@ -89,8 +77,8 @@ CreateStatistics(CreateStatsStmt *stmt, bool check_rights) bool build_dependencies; bool build_mcv; bool build_expressions; + bool has_simple_var = false; bool requested_type = false; - int i; ListCell *cell; ListCell *cell2; @@ -244,8 +232,8 @@ CreateStatistics(CreateStatsStmt *stmt, bool check_rights) STATS_MAX_DIMENSIONS))); /* - * Convert the expression list to a simple array of attnums, but also keep - * a list of more complex expressions. While at it, enforce some + * Convert the expression list to a list of expression trees. Simple + * column references are stored as Var nodes. While at it, enforce some * constraints - we don't allow extended statistics on system attributes, * and we require the data type to have a less-than operator, if we're * building multivariate statistics. @@ -300,24 +288,14 @@ CreateStatistics(CreateStatsStmt *stmt, bool check_rights) format_type_be(attForm->atttypid)))); } - /* Treat virtual generated columns as expressions */ - if (attForm->attgenerated == ATTRIBUTE_GENERATED_VIRTUAL) - { - Node *expr; - - expr = (Node *) makeVar(1, - attForm->attnum, - attForm->atttypid, - attForm->atttypmod, - attForm->attcollation, - 0); - stxexprs = lappend(stxexprs, expr); - } - else - { - attnums[nattnums] = attForm->attnum; - nattnums++; - } + stxexprs = lappend(stxexprs, + (Node *) makeVar(1, + attForm->attnum, + attForm->atttypid, + attForm->atttypmod, + attForm->attcollation, + 0)); + has_simple_var = true; ReleaseSysCache(atttuple); } else if (IsA(selem->expr, Var)) /* column reference in parens */ @@ -347,16 +325,8 @@ CreateStatistics(CreateStatsStmt *stmt, bool check_rights) format_type_be(var->vartype)))); } - /* Treat virtual generated columns as expressions */ - if (get_attgenerated(relid, var->varattno) == ATTRIBUTE_GENERATED_VIRTUAL) - { - stxexprs = lappend(stxexprs, (Node *) var); - } - else - { - attnums[nattnums] = var->varattno; - nattnums++; - } + stxexprs = lappend(stxexprs, (Node *) var); + has_simple_var = true; } else /* expression */ { @@ -407,20 +377,23 @@ CreateStatistics(CreateStatsStmt *stmt, bool check_rights) * that we're building statistics on a single expression (or virtual * generated column). */ - if (numcols < 2 && list_length(stxexprs) != 1) - ereport(ERROR, - errcode(ERRCODE_INVALID_OBJECT_DEFINITION), - errmsg("cannot create extended statistics on a single non-virtual column"), - errdetail("Univariate statistics are already built for each individual non-virtual table column.")); + if (numcols == 1) + { + Node *single = (Node *) linitial(stxexprs); - /* - * Parse the statistics kinds (not allowed when building univariate - * statistics). - */ - if (numcols == 1 && stmt->stat_types != NIL) - ereport(ERROR, - errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot specify statistics kinds when building univariate statistics")); + if (IsA(single, Var) && + get_attgenerated(relid, ((Var *) single)->varattno) != ATTRIBUTE_GENERATED_VIRTUAL) + ereport(ERROR, + errcode(ERRCODE_INVALID_OBJECT_DEFINITION), + errmsg("cannot create extended statistics on a single non-virtual column"), + errdetail("Univariate statistics are already built for each individual non-virtual table column.")); + + /* statistics kinds are not allowed with univariate statistics */ + if (stmt->stat_types != NIL) + ereport(ERROR, + errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot specify statistics kinds when building univariate statistics")); + } build_ndistinct = false; build_dependencies = false; @@ -463,50 +436,49 @@ CreateStatistics(CreateStatsStmt *stmt, bool check_rights) } /* - * When there are non-trivial expressions, build the expression stats - * automatically. This allows calculating good estimates for stats that - * consider per-clause estimates (e.g. functional dependencies). + * Build per-expression statistics automatically when the object covers + * anything that has no pg_statistic entry of its own: complex + * expressions, or virtual generated columns (stored as Var nodes, but + * like expressions they get no pg_statistic row, so they need + * per-expression stats). Plain columns are skipped, since they already + * have univariate statistics in pg_statistic. These give good estimates + * where per-clause estimates matter (e.g. functional dependencies). */ - build_expressions = (stxexprs != NIL); - - /* - * Sort the attnums, which makes detecting duplicates somewhat easier, and - * it does not hurt (it does not matter for the contents, unlike for - * indexes, for example). - */ - qsort(attnums, nattnums, sizeof(int16), compare_int16); - - /* - * Check for duplicates in the list of columns. The attnums are sorted so - * just check consecutive elements. - */ - for (i = 1; i < nattnums; i++) + build_expressions = false; + foreach(cell, stxexprs) { - if (attnums[i] == attnums[i - 1]) - ereport(ERROR, - (errcode(ERRCODE_DUPLICATE_COLUMN), - errmsg("duplicate column name in statistics definition"))); + Node *expr = (Node *) lfirst(cell); + + if (!IsA(expr, Var) || + get_attgenerated(relid, ((Var *) expr)->varattno) == ATTRIBUTE_GENERATED_VIRTUAL) + { + build_expressions = true; + break; + } } /* - * Check for duplicate expressions. We do two loops, counting the - * occurrences of each expression. This is O(N^2) but we only allow small - * number of expressions and it's not executed often. + * Check for duplicate entries. We do two loops, counting the occurrences + * of each entry. This is O(N^2) but we only allow small number of entries + * and it's not executed often. * - * XXX We don't cross-check attributes and expressions, because it does - * not seem worth it. In principle we could check that expressions don't - * contain trivial attribute references like "(a)", but the reasoning is - * similar to why we don't bother with extracting columns from - * expressions. It's either expensive or very easy to defeat for - * determined user, and there's no risk if we allow such statistics (the - * statistics is useless, but harmless). + * We compare the entries after expanding generated columns and + * const-folding, so that an entry which resolves to a column already + * listed is caught as a duplicate; a plain structural comparison would + * miss it. References that are equivalent but do not fold to the same + * node (e.g. "a" vs "(a+0)") are still not caught; such an object is + * useless but harmless. */ - foreach(cell, stxexprs) + folded_exprs = (List *) expand_generated_columns_in_expr((Node *) stxexprs, + rel, 1); + folded_exprs = (List *) eval_const_expressions(NULL, (Node *) folded_exprs); + + foreach(cell, folded_exprs) { Node *expr1 = (Node *) lfirst(cell); int cnt = 0; - foreach(cell2, stxexprs) + foreach(cell2, folded_exprs) { Node *expr2 = (Node *) lfirst(cell2); @@ -514,18 +486,15 @@ CreateStatistics(CreateStatsStmt *stmt, bool check_rights) cnt += 1; } - /* every expression should find at least itself */ + /* every entry should find at least itself */ Assert(cnt >= 1); if (cnt > 1) ereport(ERROR, (errcode(ERRCODE_DUPLICATE_COLUMN), - errmsg("duplicate expression in statistics definition"))); + errmsg("duplicate column or expression in statistics definition"))); } - /* Form an int2vector representation of the sorted column list */ - stxkeys = buildint2vector(attnums, nattnums); - /* construct the char array of enabled statistic types */ ntypes = 0; if (build_ndistinct) @@ -539,17 +508,10 @@ CreateStatistics(CreateStatsStmt *stmt, bool check_rights) Assert(ntypes > 0 && ntypes <= lengthof(types)); stxkind = construct_array_builtin(types, ntypes, CHAROID); - /* convert the expressions (if any) to a text datum */ - if (stxexprs != NIL) - { - char *exprsString; - - exprsString = nodeToString(stxexprs); - exprsDatum = CStringGetTextDatum(exprsString); - pfree(exprsString); - } - else - exprsDatum = (Datum) 0; + /* convert the expression list to a text datum */ + exprsString = nodeToString(stxexprs); + exprsDatum = CStringGetTextDatum(exprsString); + pfree(exprsString); statrel = table_open(StatisticExtRelationId, RowExclusiveLock); @@ -566,13 +528,9 @@ CreateStatistics(CreateStatsStmt *stmt, bool check_rights) values[Anum_pg_statistic_ext_stxname - 1] = NameGetDatum(&stxname); values[Anum_pg_statistic_ext_stxnamespace - 1] = ObjectIdGetDatum(namespaceId); values[Anum_pg_statistic_ext_stxowner - 1] = ObjectIdGetDatum(stxowner); - values[Anum_pg_statistic_ext_stxkeys - 1] = PointerGetDatum(stxkeys); nulls[Anum_pg_statistic_ext_stxstattarget - 1] = true; values[Anum_pg_statistic_ext_stxkind - 1] = PointerGetDatum(stxkind); - values[Anum_pg_statistic_ext_stxexprs - 1] = exprsDatum; - if (exprsDatum == (Datum) 0) - nulls[Anum_pg_statistic_ext_stxexprs - 1] = true; /* insert it into pg_statistic_ext */ htup = heap_form_tuple(statrel->rd_att, values, nulls); @@ -602,13 +560,6 @@ CreateStatistics(CreateStatsStmt *stmt, bool check_rights) */ ObjectAddressSet(myself, StatisticExtRelationId, statoid); - /* add dependencies for plain column references */ - for (i = 0; i < nattnums; i++) - { - ObjectAddressSubSet(parentobject, RelationRelationId, relid, attnums[i]); - recordDependencyOn(&myself, &parentobject, DEPENDENCY_AUTO); - } - /* * If there are no dependencies on a column, give the statistics object an * auto dependency on the whole table. In most cases, this will be @@ -620,7 +571,7 @@ CreateStatistics(CreateStatsStmt *stmt, bool check_rights) * dependency, because recordDependencyOnSingleRelExpr may not create any * dependencies for whole-row Vars. */ - if (!nattnums) + if (!has_simple_var) { ObjectAddressSet(parentobject, RelationRelationId, relid); recordDependencyOn(&myself, &parentobject, DEPENDENCY_AUTO); @@ -630,12 +581,11 @@ CreateStatistics(CreateStatsStmt *stmt, bool check_rights) * Store dependencies on anything mentioned in statistics expressions, * just like we do for index expressions. */ - if (stxexprs) - recordDependencyOnSingleRelExpr(&myself, - (Node *) stxexprs, - relid, - DEPENDENCY_NORMAL, - DEPENDENCY_AUTO, false); + recordDependencyOnSingleRelExpr(&myself, + (Node *) stxexprs, + relid, + DEPENDENCY_NORMAL, + DEPENDENCY_AUTO, false); /* * Also add dependencies on namespace and owner. These are required diff --git a/src/backend/optimizer/util/plancat.c b/src/backend/optimizer/util/plancat.c index 7c4be174869..da92d70b4af 100644 --- a/src/backend/optimizer/util/plancat.c +++ b/src/backend/optimizer/util/plancat.c @@ -1654,7 +1654,7 @@ get_relation_constraints(PlannerInfo *root, static void get_relation_statistics_worker(List **stainfos, RelOptInfo *rel, Oid statOid, bool inh, - Bitmapset *keys, List *exprs) + List *exprs) { Form_pg_statistic_ext_data dataForm; HeapTuple dtup; @@ -1675,7 +1675,6 @@ get_relation_statistics_worker(List **stainfos, RelOptInfo *rel, info->inherit = dataForm->stxdinherit; info->rel = rel; info->kind = STATS_EXT_NDISTINCT; - info->keys = bms_copy(keys); info->exprs = exprs; *stainfos = lappend(*stainfos, info); @@ -1689,7 +1688,6 @@ get_relation_statistics_worker(List **stainfos, RelOptInfo *rel, info->inherit = dataForm->stxdinherit; info->rel = rel; info->kind = STATS_EXT_DEPENDENCIES; - info->keys = bms_copy(keys); info->exprs = exprs; *stainfos = lappend(*stainfos, info); @@ -1703,7 +1701,6 @@ get_relation_statistics_worker(List **stainfos, RelOptInfo *rel, info->inherit = dataForm->stxdinherit; info->rel = rel; info->kind = STATS_EXT_MCV; - info->keys = bms_copy(keys); info->exprs = exprs; *stainfos = lappend(*stainfos, info); @@ -1717,7 +1714,6 @@ get_relation_statistics_worker(List **stainfos, RelOptInfo *rel, info->inherit = dataForm->stxdinherit; info->rel = rel; info->kind = STATS_EXT_EXPRESSIONS; - info->keys = bms_copy(keys); info->exprs = exprs; *stainfos = lappend(*stainfos, info); @@ -1748,88 +1744,57 @@ get_relation_statistics(PlannerInfo *root, RelOptInfo *rel, foreach(l, statoidlist) { Oid statOid = lfirst_oid(l); - Form_pg_statistic_ext staForm; HeapTuple htup; - Bitmapset *keys = NULL; List *exprs = NIL; - int i; htup = SearchSysCache1(STATEXTOID, ObjectIdGetDatum(statOid)); if (!HeapTupleIsValid(htup)) elog(ERROR, "cache lookup failed for statistics object %u", statOid); - staForm = (Form_pg_statistic_ext) GETSTRUCT(htup); - - /* - * First, build the array of columns covered. This is ultimately - * wasted if no stats within the object have actually been built, but - * it doesn't seem worth troubling over that case. - */ - for (i = 0; i < staForm->stxkeys.dim1; i++) - keys = bms_add_member(keys, staForm->stxkeys.values[i]); /* - * Preprocess expressions (if any). We read the expressions, fix the + * Preprocess the columns and expressions. We read them, fix the * varnos, and run them through eval_const_expressions. * * XXX We don't know yet if there are any data for this stats object, * with either stxdinherit value. But it's reasonable to assume there * is at least one of those, possibly both. So it's better to process - * keys and expressions here. + * columns and expressions here. */ - { - bool isnull; - Datum datum; - - /* decode expression (if any) */ - datum = SysCacheGetAttr(STATEXTOID, htup, - Anum_pg_statistic_ext_stxexprs, &isnull); + exprs = statext_get_stxexprs(htup, relation); - if (!isnull) - { - char *exprsString; - - exprsString = TextDatumGetCString(datum); - exprs = (List *) stringToNode(exprsString); - pfree(exprsString); - - /* Expand virtual generated columns in the expressions */ - exprs = (List *) expand_generated_columns_in_expr((Node *) exprs, relation, 1); - - /* - * Modify the copies we obtain from the relcache to have the - * correct varno for the parent relation, so that they match - * up correctly against qual clauses. - * - * This must be done before const-simplification because - * eval_const_expressions reduces NullTest for Vars based on - * varno. - */ - if (varno != 1) - ChangeVarNodes((Node *) exprs, 1, varno, 0); + /* + * Modify the copies we obtain from the relcache to have the correct + * varno for the parent relation, so that they match up correctly + * against qual clauses. + * + * This must be done before const-simplification because + * eval_const_expressions reduces NullTest for Vars based on varno. + */ + if (varno != 1) + ChangeVarNodes((Node *) exprs, 1, varno, 0); - /* - * Run the expressions through eval_const_expressions. This is - * not just an optimization, but is necessary, because the - * planner will be comparing them to similarly-processed qual - * clauses, and may fail to detect valid matches without this. - * We must not use canonicalize_qual, however, since these - * aren't qual expressions. - */ - exprs = (List *) eval_const_expressions(root, (Node *) exprs); + /* + * Run the columns and expressions through eval_const_expressions. + * This is not just an optimization, but is necessary, because the + * planner will be comparing them to similarly-processed qual clauses, + * and may fail to detect valid matches without this. We must not use + * canonicalize_qual, however, since these aren't qual expressions. + * Plain-column Vars are unaffected. statext_get_stxexprs() already + * const-folded these with a NULL root; we redo it here with the real + * root so the varno-dependent reductions noted above can apply. + */ + exprs = (List *) eval_const_expressions(root, (Node *) exprs); - /* May as well fix opfuncids too */ - fix_opfuncids((Node *) exprs); - } - } + /* May as well fix opfuncids too */ + fix_opfuncids((Node *) exprs); /* extract statistics for possible values of stxdinherit flag */ - get_relation_statistics_worker(&stainfos, rel, statOid, true, keys, exprs); + get_relation_statistics_worker(&stainfos, rel, statOid, true, exprs); - get_relation_statistics_worker(&stainfos, rel, statOid, false, keys, exprs); + get_relation_statistics_worker(&stainfos, rel, statOid, false, exprs); ReleaseSysCache(htup); - bms_free(keys); } list_free(statoidlist); diff --git a/src/backend/parser/parse_utilcmd.c b/src/backend/parser/parse_utilcmd.c index ccf6ee55310..51371b22d61 100644 --- a/src/backend/parser/parse_utilcmd.c +++ b/src/backend/parser/parse_utilcmd.c @@ -2042,9 +2042,9 @@ generateClonedIndexStmt(RangeVar *heapRel, Relation source_idx, * extended statistic "source_statsid", for the rel identified by heapRel and * heapRelid. * - * stxkeys in the source statistic holds attribute numbers from the parent + * stxexprs in the source statistic holds Var nodes referencing the parent * relation. Those attnums, along with the attribute numbers referenced by - * Vars inside the expression tree, are remapped to the new relation's + * Vars inside complex expressions, are remapped to the new relation's * numbering according to attmap. */ static CreateStatsStmt * @@ -2052,15 +2052,16 @@ generateClonedExtStatsStmt(RangeVar *heapRel, Oid heapRelid, Oid source_statsid, const AttrMap *attmap) { HeapTuple ht_stats; - Form_pg_statistic_ext statsrec; CreateStatsStmt *stats; List *stat_types = NIL; List *def_names = NIL; - bool isnull; Datum datum; ArrayType *arr; char *enabled; int i; + ListCell *lc; + List *exprs = NIL; + char *exprsString; Assert(OidIsValid(heapRelid)); Assert(heapRel != NULL); @@ -2071,7 +2072,6 @@ generateClonedExtStatsStmt(RangeVar *heapRel, Oid heapRelid, ht_stats = SearchSysCache1(STATEXTOID, ObjectIdGetDatum(source_statsid)); if (!HeapTupleIsValid(ht_stats)) elog(ERROR, "cache lookup failed for statistics object %u", source_statsid); - statsrec = (Form_pg_statistic_ext) GETSTRUCT(ht_stats); /* Determine which statistics types exist */ datum = SysCacheGetAttrNotNull(STATEXTOID, ht_stats, @@ -2097,44 +2097,31 @@ generateClonedExtStatsStmt(RangeVar *heapRel, Oid heapRelid, elog(ERROR, "unrecognized statistics kind %c", enabled[i]); } - /* Determine which columns the statistics are on */ - for (i = 0; i < statsrec->stxkeys.dim1; i++) - { - StatsElem *selem = makeNode(StatsElem); - AttrNumber attnum = statsrec->stxkeys.values[i]; - - selem->name = - get_attname(heapRelid, attmap->attnums[attnum - 1], false); - selem->expr = NULL; - - def_names = lappend(def_names, selem); - } - /* - * Now handle expressions, if there are any. The order (with respect to - * regular attributes) does not really matter for extended stats, so we - * simply append them after simple column references. - * - * XXX Some places during build/estimation treat expressions as if they - * are before attributes, but for the CREATE command that's entirely - * irrelevant. + * Decode stxexprs to reconstruct the list of columns and expressions. + * Simple Var nodes represent plain column references; other nodes are + * complex expressions. */ - datum = SysCacheGetAttr(STATEXTOID, ht_stats, - Anum_pg_statistic_ext_stxexprs, &isnull); + datum = SysCacheGetAttrNotNull(STATEXTOID, ht_stats, + Anum_pg_statistic_ext_stxexprs); + exprsString = TextDatumGetCString(datum); + exprs = (List *) stringToNode(exprsString); - if (!isnull) + foreach(lc, exprs) { - ListCell *lc; - List *exprs = NIL; - char *exprsString; + Node *expr = (Node *) lfirst(lc); + StatsElem *selem = makeNode(StatsElem); - exprsString = TextDatumGetCString(datum); - exprs = (List *) stringToNode(exprsString); + if (IsA(expr, Var) && ((Var *) expr)->varattno > 0) + { + AttrNumber attnum = ((Var *) expr)->varattno; - foreach(lc, exprs) + selem->name = + get_attname(heapRelid, attmap->attnums[attnum - 1], false); + selem->expr = NULL; + } + else { - Node *expr = (Node *) lfirst(lc); - StatsElem *selem = makeNode(StatsElem); bool found_whole_row; /* Adjust Vars to match new table's column numbering */ @@ -2146,13 +2133,13 @@ generateClonedExtStatsStmt(RangeVar *heapRel, Oid heapRelid, selem->name = NULL; selem->expr = expr; - - def_names = lappend(def_names, selem); } - pfree(exprsString); + def_names = lappend(def_names, selem); } + pfree(exprsString); + /* finally, build the output node */ stats = makeNode(CreateStatsStmt); stats->defnames = NULL; diff --git a/src/backend/statistics/dependencies.c b/src/backend/statistics/dependencies.c index 628526ebff6..2fe16adecb3 100644 --- a/src/backend/statistics/dependencies.c +++ b/src/backend/statistics/dependencies.c @@ -393,12 +393,19 @@ statext_dependencies_build(StatsBuildData *data) d = (MVDependency *) palloc0(offsetof(MVDependency, attributes) + k * sizeof(AttrNumber)); - /* copy the dependency (and keep the indexes into stxkeys) */ + /* copy the dependency */ d->degree = degree; d->nattributes = k; for (i = 0; i < k; i++) d->attributes[i] = data->attnums[dependency[i]]; + /* + * Order the first (k-1) attnums the way dump and restore needs + * (see compare_attnums); the last is the dependent attribute and + * stays last. + */ + qsort(d->attributes, k - 1, sizeof(AttrNumber), compare_attnums); + /* initialize the list of dependencies */ if (dependencies == NULL) { @@ -596,13 +603,13 @@ statext_dependencies_free(MVDependencies *dependencies) * attributes list correspond to attnums/expressions defined by the * extended statistics object. * - * Positive attnums are attributes which must be found in the stxkeys, while - * negative attnums correspond to an expression number, no attribute number - * can be below (0 - numexprs). + * Positive attnums correspond to table columns (excluding virtual generated + * columns), while negative attnums correspond to expressions. No attribute + * number can be below (0 - numexprs). */ bool statext_dependencies_validate(const MVDependencies *dependencies, - const int2vector *stxkeys, + const Bitmapset *keys, int numexprs, int elevel) { int attnum_expr_lowbound = 0 - numexprs; @@ -623,15 +630,8 @@ statext_dependencies_validate(const MVDependencies *dependencies, if (attnum > 0) { - /* attribute number in stxkeys */ - for (int k = 0; k < stxkeys->dim1; k++) - { - if (attnum == stxkeys->values[k]) - { - ok = true; - break; - } - } + /* attribute number in keys */ + ok = bms_is_member(attnum, keys); } else if ((attnum < 0) && (attnum >= attnum_expr_lowbound)) { @@ -1308,6 +1308,9 @@ dependency_is_compatible_expression(Node *clause, Index relid, List *statlist, N { Node *stat_expr = (Node *) lfirst(lc2); + if (statext_is_column(stat_expr)) + continue; + if (equal(clause_expr, stat_expr)) { *expr = stat_expr; @@ -1546,7 +1549,7 @@ dependencies_clauselist_selectivity(PlannerInfo *root, StatisticExtInfo *stat = (StatisticExtInfo *) lfirst(l); int nmatched; int nexprs; - int k; + ListCell *lc_col; MVDependencies *deps; /* skip statistics that are not of the correct type */ @@ -1558,21 +1561,22 @@ dependencies_clauselist_selectivity(PlannerInfo *root, continue; /* - * Count matching attributes - we have to undo the attnum offsets. The - * input attribute numbers are not offset (expressions are not - * included in stat->keys, so it's not necessary). But we need to - * offset it before checking against clauses_attnums. + * Count matching attributes. We walk the covered plain columns + * (expression entries are skipped) and offset each attnum before + * checking it against clauses_attnums. */ nmatched = 0; - k = -1; - while ((k = bms_next_member(stat->keys, k)) >= 0) + foreach(lc_col, stat->exprs) { - AttrNumber attnum = (AttrNumber) k; + Node *node = (Node *) lfirst(lc_col); + AttrNumber attnum; - /* skip expressions */ - if (!AttrNumberIsForUserDefinedAttr(attnum)) + /* only plain columns are counted here */ + if (!statext_is_column(node)) continue; + attnum = ((Var *) node)->varattno; + /* apply the same offset as above */ attnum += attnum_offset; @@ -1590,6 +1594,9 @@ dependencies_clauselist_selectivity(PlannerInfo *root, { Node *stat_expr = (Node *) lfirst(lc); + if (statext_is_column(stat_expr)) + continue; + /* try to match it */ if (equal(stat_expr, unique_exprs[i])) nexprs++; @@ -1631,7 +1638,7 @@ dependencies_clauselist_selectivity(PlannerInfo *root, * it in a cheaper way (if there are no expr clauses, we can just * discard all negative attnums without any lookups). */ - if (unique_exprs_cnt > 0 || stat->exprs != NIL) + if (unique_exprs_cnt > 0 || stat_num_expressions(stat) > 0) { uint32 ndeps = 0; @@ -1686,9 +1693,9 @@ dependencies_clauselist_selectivity(PlannerInfo *root, idx = -(1 + attnum); /* Is the expression index is valid? */ - Assert((idx >= 0) && (idx < list_length(stat->exprs))); + Assert((idx >= 0) && (idx < stat_num_expressions(stat))); - expr = (Node *) list_nth(stat->exprs, idx); + expr = stat_nth_expression(stat, idx); /* try to find the expression in the unique list */ for (int m = 0; m < unique_exprs_cnt; m++) diff --git a/src/backend/statistics/extended_stats.c b/src/backend/statistics/extended_stats.c index e6e483ceea0..898812682d8 100644 --- a/src/backend/statistics/extended_stats.c +++ b/src/backend/statistics/extended_stats.c @@ -67,15 +67,15 @@ typedef struct StatExtEntry Oid statOid; /* OID of pg_statistic_ext entry */ char *schema; /* statistics object's schema */ char *name; /* statistics object's name */ - Bitmapset *columns; /* attribute numbers covered by the object */ List *types; /* 'char' list of enabled statistics kinds */ int stattarget; /* statistics target (-1 for default) */ - List *exprs; /* expressions */ + List *exprs; /* all columns and expressions in + * user-declared order, mirroring stxexprs */ } StatExtEntry; static List *fetch_statentries_for_relation(Relation pg_statext, Relation rel); -static VacAttrStats **lookup_var_attr_stats(Bitmapset *attrs, List *exprs, +static VacAttrStats **lookup_var_attr_stats(List *exprs, int nvacatts, VacAttrStats **vacatts); static void statext_store(Oid statOid, bool inh, MVNDistinct *ndistinct, MVDependencies *dependencies, @@ -166,8 +166,7 @@ BuildRelationExtStatistics(Relation onerel, bool inh, double totalrows, * Check if we can build these stats based on the column analyzed. If * not, report this fact (except in autovacuum) and move on. */ - stats = lookup_var_attr_stats(stat->columns, stat->exprs, - natts, vacattrstats); + stats = lookup_var_attr_stats(stat->exprs, natts, vacattrstats); if (!stats) { if (!AmAutoVacuumWorkerProcess()) @@ -183,7 +182,7 @@ BuildRelationExtStatistics(Relation onerel, bool inh, double totalrows, /* compute statistics target for this statistics object */ stattarget = statext_compute_stattarget(stat->stattarget, - bms_num_members(stat->columns), + list_length(stat->exprs), stats); /* @@ -212,17 +211,40 @@ BuildRelationExtStatistics(Relation onerel, bool inh, double totalrows, { AnlExprData *exprdata; int nexprs; + List *cmplxexprs = NIL; + ListCell *lc3; - /* should not happen, thanks to checks when defining stats */ - if (!stat->exprs) - elog(ERROR, "requested expression stats, but there are no expressions"); + /* + * stat->exprs holds all columns and expressions in declared + * order; expression stats cover only the complex expressions, + * so filter out the plain Var entries. + */ + foreach(lc3, stat->exprs) + { + Node *node = (Node *) lfirst(lc3); + + if (statext_is_column(node)) + continue; + + cmplxexprs = lappend(cmplxexprs, node); + } - exprdata = build_expr_data(stat->exprs, stattarget); - nexprs = list_length(stat->exprs); + /* + * Build per-expression stats only if any complex expressions + * remain after folding; if every entry reduced to a plain + * column there is nothing to compute, and stxdexpr is left + * NULL (like any other kind whose build finds nothing; see + * statext_store). + */ + if (cmplxexprs != NIL) + { + exprdata = build_expr_data(cmplxexprs, stattarget); + nexprs = list_length(cmplxexprs); - compute_expr_stats(onerel, exprdata, nexprs, rows, numrows); + compute_expr_stats(onerel, exprdata, nexprs, rows, numrows); - exprstats = serialize_expr_stats(exprdata, nexprs); + exprstats = serialize_expr_stats(exprdata, nexprs); + } } } @@ -321,15 +343,14 @@ ComputeExtStatisticsRows(Relation onerel, StatExtEntry *stat = (StatExtEntry *) lfirst(lc); int stattarget; VacAttrStats **stats; - int nattrs = bms_num_members(stat->columns); + int nattrs = list_length(stat->exprs); /* * Check if we can build this statistics object based on the columns * analyzed. If not, ignore it (don't report anything, we'll do that * during the actual build BuildRelationExtStatistics). */ - stats = lookup_var_attr_stats(stat->columns, stat->exprs, - natts, vacattrstats); + stats = lookup_var_attr_stats(stat->exprs, natts, vacattrstats); if (!stats) continue; @@ -447,6 +468,148 @@ statext_is_kind_built(HeapTuple htup, char type) return !heap_attisnull(htup, attnum, NULL); } +/* + * statext_get_stxexprs + * Decode the stxexprs field of a pg_statistic_ext tuple into the full + * list of columns and expressions, in user-declared order. + * + * Deserializes the expression list, expands virtual generated columns, and + * const-folds the whole list (as RelationGetIndexExpressions does). The + * returned list contains all of them, both simple Var references (plain + * columns) and complex expressions, in the order they were declared in + * CREATE STATISTICS, mirroring the catalog stxexprs. + * + * Const-folding the whole list (rather than only the complex expressions) is + * harmless, because plain Var nodes are unaffected by eval_const_expressions. + */ +List * +statext_get_stxexprs(HeapTuple htup, Relation rel) +{ + Datum datum; + char *exprsString; + List *allexprs; + + datum = SysCacheGetAttrNotNull(STATEXTOID, htup, + Anum_pg_statistic_ext_stxexprs); + exprsString = TextDatumGetCString(datum); + allexprs = (List *) stringToNode(exprsString); + pfree(exprsString); + + /* Expand virtual generated columns in the expressions */ + allexprs = (List *) expand_generated_columns_in_expr((Node *) allexprs, rel, 1); + + /* + * Run the expressions through eval_const_expressions. This is not just an + * optimization, but is necessary, because the planner will be comparing + * them to similarly-processed qual clauses, and may fail to detect valid + * matches without this. We must not use canonicalize_qual, however, + * since these aren't qual expressions. + */ + allexprs = (List *) eval_const_expressions(NULL, (Node *) allexprs); + + /* May as well fix opfuncids too */ + fix_opfuncids((Node *) allexprs); + + return allexprs; +} + +/* + * statext_is_column + * Is a statistics-object entry a plain column rather than an expression? + * Non-Vars and whole-row Vars (attnum 0) are expressions. Callers pass + * lists from statext_get_stxexprs(), which expands generated columns and + * const-folds, so a positive-attnum Var here may come from a folded + * expression or a trivial virtual generated column (e.g. "CASE WHEN true + * THEN a END" -> "a"). Either way it resolves to that plain column, whose + * statistics live in pg_statistic, so treating it as a column is correct. + */ +bool +statext_is_column(Node *node) +{ + return IsA(node, Var) && + AttrNumberIsForUserDefinedAttr(((Var *) node)->varattno); +} + +/* + * stat_covers_attnum + * Is attnum one of the statistics object's columns? + */ +bool +stat_covers_attnum(StatisticExtInfo *stat, AttrNumber attnum) +{ + ListCell *lc; + + foreach(lc, stat->exprs) + { + Node *node = (Node *) lfirst(lc); + + if (statext_is_column(node) && + ((Var *) node)->varattno == attnum) + return true; + } + return false; +} + +/* + * stat_covers_attnums + * Does the statistics object cover every attnum in the set? + * An empty set is trivially covered. + */ +static bool +stat_covers_attnums(StatisticExtInfo *stat, Bitmapset *attnums) +{ + int k = -1; + + while ((k = bms_next_member(attnums, k)) >= 0) + { + if (!stat_covers_attnum(stat, (AttrNumber) k)) + return false; + } + return true; +} + +/* + * stat_num_expressions + * Number of expressions (non-column entries) of the statistics object. + */ +int +stat_num_expressions(StatisticExtInfo *stat) +{ + ListCell *lc; + int n = 0; + + foreach(lc, stat->exprs) + { + if (!statext_is_column((Node *) lfirst(lc))) + n++; + } + return n; +} + +/* + * stat_nth_expression + * Return the n-th expression (non-column entry), in the per-expression + * ordering used by the stored stxdexpr statistics. + */ +Node * +stat_nth_expression(StatisticExtInfo *stat, int n) +{ + ListCell *lc; + int idx = 0; + + foreach(lc, stat->exprs) + { + Node *node = (Node *) lfirst(lc); + + if (statext_is_column(node)) + continue; + if (idx == n) + return node; + idx++; + } + return NULL; +} + /* * Return a list (of StatExtEntry) of statistics objects for the given relation. */ @@ -480,18 +643,12 @@ fetch_statentries_for_relation(Relation pg_statext, Relation rel) ArrayType *arr; char *enabled; Form_pg_statistic_ext staForm; - List *exprs = NIL; entry = palloc0_object(StatExtEntry); staForm = (Form_pg_statistic_ext) GETSTRUCT(htup); entry->statOid = staForm->oid; entry->schema = get_namespace_name(staForm->stxnamespace); entry->name = pstrdup(NameStr(staForm->stxname)); - for (i = 0; i < staForm->stxkeys.dim1; i++) - { - entry->columns = bms_add_member(entry->columns, - staForm->stxkeys.values[i]); - } datum = SysCacheGetAttr(STATEXTOID, htup, Anum_pg_statistic_ext_stxstattarget, &isnull); entry->stattarget = isnull ? -1 : DatumGetInt16(datum); @@ -514,37 +671,11 @@ fetch_statentries_for_relation(Relation pg_statext, Relation rel) entry->types = lappend_int(entry->types, (int) enabled[i]); } - /* decode expression (if any) */ - datum = SysCacheGetAttr(STATEXTOID, htup, - Anum_pg_statistic_ext_stxexprs, &isnull); - - if (!isnull) - { - char *exprsString; - - exprsString = TextDatumGetCString(datum); - exprs = (List *) stringToNode(exprsString); - - pfree(exprsString); - - /* Expand virtual generated columns in the expressions */ - exprs = (List *) expand_generated_columns_in_expr((Node *) exprs, rel, 1); - - /* - * Run the expressions through eval_const_expressions. This is not - * just an optimization, but is necessary, because the planner - * will be comparing them to similarly-processed qual clauses, and - * may fail to detect valid matches without this. We must not use - * canonicalize_qual, however, since these aren't qual - * expressions. - */ - exprs = (List *) eval_const_expressions(NULL, (Node *) exprs); - - /* May as well fix opfuncids too */ - fix_opfuncids((Node *) exprs); - } - - entry->exprs = exprs; + /* + * Decode stxexprs into all columns and expressions, in declared + * order. + */ + entry->exprs = statext_get_stxexprs(htup, rel); result = lappend(result, entry); } @@ -720,79 +851,78 @@ examine_expression(Node *expr, int stattarget) /* * Using 'vacatts' of size 'nvacatts' as input data, return a newly-built - * VacAttrStats array which includes only the items corresponding to - * attributes indicated by 'attrs'. If we don't have all of the per-column - * stats available to compute the extended stats, then we return NULL to - * indicate to the caller that the stats should not be built. + * VacAttrStats array which includes only the items corresponding to the + * columns and expressions in 'exprs', in the same order. If we don't have all + * of the per-column stats available to compute the extended stats, then we + * return NULL to indicate to the caller that the stats should not be built. */ static VacAttrStats ** -lookup_var_attr_stats(Bitmapset *attrs, List *exprs, - int nvacatts, VacAttrStats **vacatts) +lookup_var_attr_stats(List *exprs, int nvacatts, VacAttrStats **vacatts) { int i = 0; - int x = -1; int natts; VacAttrStats **stats; ListCell *lc; - natts = bms_num_members(attrs) + list_length(exprs); + natts = list_length(exprs); stats = (VacAttrStats **) palloc(natts * sizeof(VacAttrStats *)); - /* lookup VacAttrStats info for the requested columns (same attnum) */ - while ((x = bms_next_member(attrs, x)) >= 0) + foreach(lc, exprs) { - int j; + Node *node = (Node *) lfirst(lc); - stats[i] = NULL; - for (j = 0; j < nvacatts; j++) + if (statext_is_column(node)) { - if (x == vacatts[j]->tupattnum) + AttrNumber attno = ((Var *) node)->varattno; + int j; + + /* lookup VacAttrStats info for a plain column (same attnum) */ + stats[i] = NULL; + for (j = 0; j < nvacatts; j++) { - stats[i] = vacatts[j]; - break; + if (attno == vacatts[j]->tupattnum) + { + stats[i] = vacatts[j]; + break; + } } - } - if (!stats[i]) + if (!stats[i]) + { + /* + * Looks like stats were not gathered for one of the columns + * required. We'll be unable to build the extended stats + * without this column. + */ + pfree(stats); + return NULL; + } + } + else { + /* an expression */ + stats[i] = examine_attribute(node); + /* - * Looks like stats were not gathered for one of the columns - * required. We'll be unable to build the extended stats without - * this column. + * If the expression has been found as non-analyzable, give up. We + * will not be able to build extended stats with it. */ - pfree(stats); - return NULL; - } - - i++; - } - - /* also add info for expressions */ - foreach(lc, exprs) - { - Node *expr = (Node *) lfirst(lc); - - stats[i] = examine_attribute(expr); + if (stats[i] == NULL) + { + pfree(stats); + return NULL; + } - /* - * If the expression has been found as non-analyzable, give up. We - * will not be able to build extended stats with it. - */ - if (stats[i] == NULL) - { - pfree(stats); - return NULL; + /* + * XXX We need tuple descriptor later, and we just grab it from + * stats[0]->tupDesc (see e.g. statext_mcv_build). But as coded + * examine_attribute does not set that, so just grab it from the + * first vacatts element. + */ + stats[i]->tupDesc = vacatts[0]->tupDesc; } - /* - * XXX We need tuple descriptor later, and we just grab it from - * stats[0]->tupDesc (see e.g. statext_mcv_build). But as coded - * examine_attribute does not set that, so just grab it from the first - * vacatts element. - */ - stats[i]->tupDesc = vacatts[0]->tupDesc; - i++; } @@ -976,49 +1106,33 @@ compare_datums_simple(Datum a, Datum b, SortSupport ssup) } /* - * build_attnums_array - * Transforms a bitmap into an array of AttrNumber values. + * compare_attnums + * qsort comparator that orders attribute numbers with columns (positive + * attnums) before expressions (negative attnums), columns in ascending + * and expressions in descending order. * - * This is used for extended statistics only, so all the attributes must be - * user-defined. That means offsetting by FirstLowInvalidHeapAttributeNumber - * is not necessary here (and when querying the bitmap). + * The order of the attribute numbers within an ndistinct item or a dependency + * does not matter for estimation, which looks them up by value. It matters + * only for pg_dump and pg_upgrade: they store the statistics as text and load + * it back through the pg_ndistinct and pg_dependencies input functions, which + * accept the attribute numbers only in this order. */ -AttrNumber * -build_attnums_array(Bitmapset *attrs, int nexprs, int *numattrs) +int +compare_attnums(const void *a, const void *b) { - int i, - j; - AttrNumber *attnums; - int num = bms_num_members(attrs); - - if (numattrs) - *numattrs = num; - - /* build attnums from the bitmapset */ - attnums = palloc_array(AttrNumber, num); - i = 0; - j = -1; - while ((j = bms_next_member(attrs, j)) >= 0) - { - int attnum = (j - nexprs); - - /* - * Make sure the bitmap contains only user-defined attributes. As - * bitmaps can't contain negative values, this can be violated in two - * ways. Firstly, the bitmap might contain 0 as a member, and secondly - * the integer value might be larger than MaxAttrNumber. - */ - Assert(AttributeNumberIsValid(attnum)); - Assert(attnum <= MaxAttrNumber); - Assert(attnum >= (-nexprs)); - - attnums[i++] = (AttrNumber) attnum; - - /* protect against overflows */ - Assert(i <= num); - } - - return attnums; + AttrNumber x = *(const AttrNumber *) a; + AttrNumber y = *(const AttrNumber *) b; + + /* columns sort ahead of expressions */ + if ((x > 0) && (y < 0)) + return -1; + if ((x < 0) && (y > 0)) + return 1; + + /* columns ascending, expressions descending */ + if (x > 0) + return x - y; + return y - x; } /* @@ -1195,6 +1309,10 @@ stat_find_expression(StatisticExtInfo *stat, Node *expr) { Node *stat_expr = (Node *) lfirst(lc); + /* columns have no per-expression stats, so count expressions only */ + if (statext_is_column(stat_expr)) + continue; + if (equal(stat_expr, expr)) return idx; idx++; @@ -1296,7 +1414,7 @@ choose_best_statistics(List *stats, char requiredkind, bool inh, continue; /* ignore clauses that are not covered by this object */ - if (!bms_is_subset(clause_attnums[i], info->keys) || + if (!stat_covers_attnums(info, clause_attnums[i]) || !stat_covers_expressions(info, clause_exprs[i], &expr_idxs)) continue; @@ -1314,7 +1432,7 @@ choose_best_statistics(List *stats, char requiredkind, bool inh, * save the actual number of keys in the stats so that we can choose * the narrowest stats with the most matching keys. */ - numkeys = bms_num_members(info->keys) + list_length(info->exprs); + numkeys = list_length(info->exprs); /* * Use this object when it increases the number of matched attributes @@ -1830,7 +1948,7 @@ statext_mcv_clauselist_selectivity(PlannerInfo *root, List *clauses, int varReli * This also eliminates already estimated clauses - both those * estimated before and during applying extended statistics. * - * XXX This check is needed because both bms_is_subset and + * XXX This check is needed because both stat_covers_attnums and * stat_covers_expressions return true for empty attnums and * expressions. */ @@ -1845,7 +1963,7 @@ statext_mcv_clauselist_selectivity(PlannerInfo *root, List *clauses, int varReli * We need to check both attributes and expressions, and reject if * either is not covered. */ - if (!bms_is_subset(list_attnums[listidx], stat->keys) || + if (!stat_covers_attnums(stat, list_attnums[listidx]) || !stat_covers_expressions(stat, list_exprs[listidx], NULL)) continue; @@ -2513,7 +2631,8 @@ make_build_data(Relation rel, StatExtEntry *stat, int numrows, HeapTuple *rows, EState *estate; ExprContext *econtext; List *exprstates = NIL; - int nkeys = bms_num_members(stat->columns) + list_length(stat->exprs); + List *cmplxexprs = NIL; + int nkeys = list_length(stat->exprs); ListCell *lc; /* allocate everything as a single chunk, so we can free it easily */ @@ -2566,42 +2685,29 @@ make_build_data(Relation rel, StatExtEntry *stat, int numrows, HeapTuple *rows, result->nattnums = nkeys; result->numrows = numrows; - /* fill the attribute info - first attributes, then expressions */ + /* fill the attribute info for each column and expression */ idx = 0; - k = -1; - while ((k = bms_next_member(stat->columns, k)) >= 0) - { - result->attnums[idx] = k; - result->stats[idx] = stats[idx]; - - idx++; - } - k = -1; foreach(lc, stat->exprs) { - Node *expr = (Node *) lfirst(lc); - - result->attnums[idx] = k; - result->stats[idx] = examine_expression(expr, stattarget); + Node *node = (Node *) lfirst(lc); - idx++; - k--; - } - - /* first extract values for all the regular attributes */ - for (i = 0; i < numrows; i++) - { - idx = 0; - k = -1; - while ((k = bms_next_member(stat->columns, k)) >= 0) + if (statext_is_column(node)) { - result->values[idx][i] = heap_getattr(rows[i], k, - result->stats[idx]->tupDesc, - &result->nulls[idx][i]); + result->attnums[idx] = ((Var *) node)->varattno; + result->stats[idx] = stats[idx]; + } + else + { + result->attnums[idx] = k; + result->stats[idx] = examine_expression(node, stattarget); - idx++; + cmplxexprs = lappend(cmplxexprs, node); + + k--; } + + idx++; } /* Need an EState for evaluation expressions. */ @@ -2615,11 +2721,13 @@ make_build_data(Relation rel, StatExtEntry *stat, int numrows, HeapTuple *rows, /* Arrange for econtext's scan tuple to be the tuple under test */ econtext->ecxt_scantuple = slot; - /* Set up expression evaluation state */ - exprstates = ExecPrepareExprList(stat->exprs, estate); + /* Set up expression evaluation state, for the complex expressions only */ + exprstates = ExecPrepareExprList(cmplxexprs, estate); for (i = 0; i < numrows; i++) { + ListCell *lc_state = list_head(exprstates); + /* * Reset the per-tuple context each time, to reclaim any cruft left * behind by evaluating the statistics object expressions. @@ -2629,32 +2737,44 @@ make_build_data(Relation rel, StatExtEntry *stat, int numrows, HeapTuple *rows, /* Set up for expression evaluation */ ExecStoreHeapTuple(rows[i], slot, false); - idx = bms_num_members(stat->columns); - foreach(lc, exprstates) + idx = 0; + foreach(lc, stat->exprs) { - Datum datum; - bool isnull; - ExprState *exprstate = (ExprState *) lfirst(lc); + Node *node = (Node *) lfirst(lc); - /* - * Avoid accumulating per-row evaluation memory in the - * long-lived build context. - */ - datum = ExecEvalExprSwitchContext(exprstate, - GetPerTupleExprContext(estate), - &isnull); - if (isnull) + if (statext_is_column(node)) { - result->values[idx][i] = (Datum) 0; - result->nulls[idx][i] = true; + result->values[idx][i] = + heap_getattr(rows[i], ((Var *) node)->varattno, + result->stats[idx]->tupDesc, + &result->nulls[idx][i]); } else { - result->values[idx][i] = - datumCopy(datum, - result->stats[idx]->attrtype->typbyval, - result->stats[idx]->attrtype->typlen); - result->nulls[idx][i] = false; + Datum datum; + bool isnull; + ExprState *exprstate = (ExprState *) lfirst(lc_state); + + lc_state = lnext(exprstates, lc_state); + + /* + * Avoid accumulating per-row evaluation memory in the + * long-lived build context. + */ + datum = ExecEvalExprSwitchContext(exprstate, econtext, &isnull); + if (isnull) + { + result->values[idx][i] = (Datum) 0; + result->nulls[idx][i] = true; + } + else + { + result->values[idx][i] = + datumCopy(datum, + result->stats[idx]->attrtype->typbyval, + result->stats[idx]->attrtype->typlen); + result->nulls[idx][i] = false; + } } idx++; diff --git a/src/backend/statistics/extended_stats_funcs.c b/src/backend/statistics/extended_stats_funcs.c index a3e56933b91..d4e24c3af42 100644 --- a/src/backend/statistics/extended_stats_funcs.c +++ b/src/backend/statistics/extended_stats_funcs.c @@ -17,6 +17,7 @@ #include "postgres.h" #include "access/heapam.h" +#include "access/table.h" #include "catalog/indexing.h" #include "catalog/namespace.h" #include "catalog/pg_collation_d.h" @@ -146,10 +147,8 @@ static void upsert_pg_statistic_ext_data(const Datum *values, static bool check_mcvlist_array(const ArrayType *arr, int argindex, int required_ndims, int mcv_length); -static Datum import_expressions(Relation pgsd, int numexprs, - Oid *atttypids, int32 *atttypmods, - Oid *atttypcolls, Jsonb *exprs_jsonb, - bool *exprs_is_perfect); +static Datum import_expressions(Relation pgsd, List *exprnodes, + Jsonb *exprs_jsonb, bool *exprs_is_perfect); static Datum import_mcv(const ArrayType *mcv_arr, const ArrayType *freqs_arr, const ArrayType *base_freqs_arr, @@ -331,10 +330,10 @@ extended_statistics_update(FunctionCallInfo fcinfo) bool nulls[Natts_pg_statistic_ext_data] = {0}; bool replaces[Natts_pg_statistic_ext_data] = {0}; bool success = true; - Datum exprdatum; - bool isnull; + Relation rel; List *exprs = NIL; - int numattnums = 0; + Bitmapset *keys = NULL; + ListCell *lc; int numexprs = 0; int numattrs = 0; @@ -342,6 +341,9 @@ extended_statistics_update(FunctionCallInfo fcinfo) Oid *atttypids = NULL; int32 *atttypmods = NULL; Oid *atttypcolls = NULL; + + /* the expressions (declared order), for import_expressions */ + List *exprnodes = NIL; Oid relid; Oid locked_table = InvalidOid; @@ -439,40 +441,25 @@ extended_statistics_update(FunctionCallInfo fcinfo) /* Find out what extended statistics kinds we should expect. */ expand_stxkind(tup, &enabled); - numattnums = stxform->stxkeys.dim1; - - /* decode expression (if any) */ - exprdatum = SysCacheGetAttr(STATEXTOID, - tup, - Anum_pg_statistic_ext_stxexprs, - &isnull); - if (!isnull) - { - char *s; - s = TextDatumGetCString(exprdatum); - exprs = (List *) stringToNode(s); - pfree(s); - - /* - * Run the expressions through eval_const_expressions(). This is not - * just an optimization, but is necessary, because the planner will be - * comparing them to similarly-processed qual clauses, and may fail to - * detect valid matches without this. - * - * We must not use canonicalize_qual(), however, since these are not - * qual expressions. - */ - exprs = (List *) eval_const_expressions(NULL, (Node *) exprs); + /* + * Decode stxexprs, and collect the column attnums and expression count + * needed to validate the ndistinct and dependencies inputs. + */ + rel = table_open(relid, NoLock); + exprs = statext_get_stxexprs(tup, rel); + table_close(rel, NoLock); - /* May as well fix opfuncids too */ - fix_opfuncids((Node *) exprs); + foreach(lc, exprs) + { + Node *node = (Node *) lfirst(lc); - /* Compute the number of expression, for input validation. */ - numexprs = list_length(exprs); + if (statext_is_column(node)) + keys = bms_add_member(keys, ((Var *) node)->varattno); + else + numexprs++; } - - numattrs = numattnums + numexprs; + numattrs = list_length(exprs); /* * If the object cannot support ndistinct, we should not have data for it. @@ -580,54 +567,53 @@ extended_statistics_update(FunctionCallInfo fcinfo) */ if (has.mcv || has.expressions) { + int idx; + atttypids = palloc0_array(Oid, numattrs); atttypmods = palloc0_array(int32, numattrs); atttypcolls = palloc0_array(Oid, numattrs); /* - * The leading stxkeys are attribute numbers up through numattnums. - * These keys must be in ascending AttrNumber order, but we do not - * rely on that. + * Get the type info for each dimension, in the same declared order as + * the stored MCV values. import_expressions() needs only the + * expressions, so collect those nodes as we go. */ - for (int i = 0; i < numattnums; i++) + idx = 0; + foreach(lc, exprs) { - AttrNumber attnum = stxform->stxkeys.values[i]; - HeapTuple atup = SearchSysCache2(ATTNUM, - ObjectIdGetDatum(relid), - Int16GetDatum(attnum)); - - Form_pg_attribute attr; + Node *node = (Node *) lfirst(lc); - /* Attribute not found */ - if (!HeapTupleIsValid(atup)) - elog(ERROR, "stxkeys references nonexistent attnum %d", attnum); + if (statext_is_column(node)) + { + AttrNumber attnum = ((Var *) node)->varattno; + HeapTuple atup = SearchSysCache2(ATTNUM, + ObjectIdGetDatum(relid), + Int16GetDatum(attnum)); + Form_pg_attribute attr; - attr = (Form_pg_attribute) GETSTRUCT(atup); + /* Attribute not found */ + if (!HeapTupleIsValid(atup)) + elog(ERROR, "stxexprs references nonexistent attnum %d", attnum); - if (attr->attisdropped) - elog(ERROR, "stxkeys references dropped attnum %d", attnum); + attr = (Form_pg_attribute) GETSTRUCT(atup); - atttypids[i] = attr->atttypid; - atttypmods[i] = attr->atttypmod; - atttypcolls[i] = attr->attcollation; - ReleaseSysCache(atup); - } + if (attr->attisdropped) + elog(ERROR, "stxexprs references dropped attnum %d", attnum); - /* - * After all the positive number attnums in stxkeys come the negative - * numbers (if any) which represent expressions in the order that they - * appear in stxdexpr. Because the expressions are always - * monotonically decreasing from -1, there is no point in looking at - * the values in stxkeys, it's enough to know how many of them there - * are. - */ - for (int i = numattnums; i < numattrs; i++) - { - Node *expr = list_nth(exprs, i - numattnums); + atttypids[idx] = attr->atttypid; + atttypmods[idx] = attr->atttypmod; + atttypcolls[idx] = attr->attcollation; + ReleaseSysCache(atup); + } + else + { + atttypids[idx] = exprType(node); + atttypmods[idx] = exprTypmod(node); + atttypcolls[idx] = exprCollation(node); - atttypids[i] = exprType(expr); - atttypmods[i] = exprTypmod(expr); - atttypcolls[i] = exprCollation(expr); + exprnodes = lappend(exprnodes, node); + } + idx++; } } @@ -659,7 +645,7 @@ extended_statistics_update(FunctionCallInfo fcinfo) bytea *data = DatumGetByteaPP(ndistinct_datum); MVNDistinct *ndistinct = statext_ndistinct_deserialize(data); - if (statext_ndistinct_validate(ndistinct, &stxform->stxkeys, + if (statext_ndistinct_validate(ndistinct, keys, numexprs, WARNING)) { values[Anum_pg_statistic_ext_data_stxdndistinct - 1] = ndistinct_datum; @@ -678,7 +664,7 @@ extended_statistics_update(FunctionCallInfo fcinfo) bytea *data = DatumGetByteaPP(dependencies_datum); MVDependencies *dependencies = statext_dependencies_deserialize(data); - if (statext_dependencies_validate(dependencies, &stxform->stxkeys, + if (statext_dependencies_validate(dependencies, keys, numexprs, WARNING)) { values[Anum_pg_statistic_ext_data_stxddependencies - 1] = dependencies_datum; @@ -721,18 +707,7 @@ extended_statistics_update(FunctionCallInfo fcinfo) pgsd = table_open(StatisticRelationId, RowExclusiveLock); - /* - * Generate the expressions array. - * - * The atttypids, atttypmods, and atttypcolls arrays have all the - * regular attributes listed first, so we can pass those arrays with a - * start point after the last regular attribute. There are numexprs - * elements remaining. - */ - datum = import_expressions(pgsd, numexprs, - &atttypids[numattnums], - &atttypmods[numattnums], - &atttypcolls[numattnums], + datum = import_expressions(pgsd, exprnodes, PG_GETARG_JSONB_P(EXPRESSIONS_ARG), &ok); @@ -1559,11 +1534,10 @@ pg_statistic_error: * This datum is needed to fill out a complete pg_statistic_ext_data tuple. */ static Datum -import_expressions(Relation pgsd, int numexprs, - Oid *atttypids, int32 *atttypmods, - Oid *atttypcolls, Jsonb *exprs_jsonb, - bool *exprs_is_perfect) +import_expressions(Relation pgsd, List *exprnodes, + Jsonb *exprs_jsonb, bool *exprs_is_perfect) { + int numexprs = list_length(exprnodes); const char *argname = extarginfo[EXPRESSIONS_ARG].argname; Oid pgstypoid = get_rel_type_id(StatisticRelationId); ArrayBuildState *astate = NULL; @@ -1625,12 +1599,13 @@ import_expressions(Relation pgsd, int numexprs, case jbvBinary: { bool sta_ok = false; + Node *node = (Node *) list_nth(exprnodes, i); /* a real stats object */ pgstdat = import_pg_statistic(pgsd, elem->val.binary.data, exprattnum, &array_in_fn, - atttypids[i], atttypmods[i], - atttypcolls[i], &sta_ok); + exprType(node), exprTypmod(node), + exprCollation(node), &sta_ok); /* * If some incorrect data has been found, assign NULL for diff --git a/src/backend/statistics/mcv.c b/src/backend/statistics/mcv.c index 3e90f600ebe..9a19ec02938 100644 --- a/src/backend/statistics/mcv.c +++ b/src/backend/statistics/mcv.c @@ -1529,48 +1529,37 @@ pg_mcv_list_send(PG_FUNCTION_ARGS) * Optionally determines the collation. */ static int -mcv_match_expression(Node *expr, Bitmapset *keys, List *exprs, Oid *collid) +mcv_match_expression(Node *expr, List *exprs, Oid *collid) { - int idx; + bool expr_is_var = IsA(expr, Var); + int idx = 0; + ListCell *lc; - if (IsA(expr, Var)) - { - /* simple Var, so just lookup using varattno */ - Var *var = (Var *) expr; - - if (collid) - *collid = var->varcollid; - - idx = bms_member_index(keys, var->varattno); + if (collid) + *collid = expr_is_var ? ((Var *) expr)->varcollid : exprCollation(expr); - if (idx < 0) - elog(ERROR, "variable not found in statistics object"); - } - else + foreach(lc, exprs) { - /* expression - lookup in stats expressions */ - ListCell *lc; + Node *node = (Node *) lfirst(lc); - if (collid) - *collid = exprCollation(expr); - - /* expressions are stored after the simple columns */ - idx = bms_num_members(keys); - foreach(lc, exprs) + if (expr_is_var) { - Node *stat_expr = (Node *) lfirst(lc); - - if (equal(expr, stat_expr)) - break; - - idx++; + if (IsA(node, Var) && + ((Var *) node)->varattno == ((Var *) expr)->varattno) + return idx; } + else if (equal(expr, node)) + return idx; - if (lc == NULL) - elog(ERROR, "expression not found in statistics object"); + idx++; } - return idx; + if (expr_is_var) + elog(ERROR, "variable not found in statistics object"); + else + elog(ERROR, "expression not found in statistics object"); + + return -1; /* keep compiler quiet */ } /* @@ -1594,7 +1583,7 @@ mcv_match_expression(Node *expr, Bitmapset *keys, List *exprs, Oid *collid) */ static bool * mcv_get_match_bitmap(PlannerInfo *root, List *clauses, - Bitmapset *keys, List *exprs, + List *exprs, MCVList *mcvlist, bool is_or) { ListCell *l; @@ -1644,7 +1633,7 @@ mcv_get_match_bitmap(PlannerInfo *root, List *clauses, elog(ERROR, "incompatible clause"); /* match the attribute/expression to a dimension of the statistic */ - idx = mcv_match_expression(clause_expr, keys, exprs, &collid); + idx = mcv_match_expression(clause_expr, exprs, &collid); /* * Walk through the MCV items and evaluate the current clause. We @@ -1751,7 +1740,7 @@ mcv_get_match_bitmap(PlannerInfo *root, List *clauses, } /* match the attribute/expression to a dimension of the statistic */ - idx = mcv_match_expression(clause_expr, keys, exprs, &collid); + idx = mcv_match_expression(clause_expr, exprs, &collid); /* * Walk through the MCV items and evaluate the current clause. We @@ -1822,7 +1811,7 @@ mcv_get_match_bitmap(PlannerInfo *root, List *clauses, Node *clause_expr = (Node *) (expr->arg); /* match the attribute/expression to a dimension of the statistic */ - int idx = mcv_match_expression(clause_expr, keys, exprs, NULL); + int idx = mcv_match_expression(clause_expr, exprs, NULL); /* * Walk through the MCV items and evaluate the current clause. We @@ -1864,7 +1853,7 @@ mcv_get_match_bitmap(PlannerInfo *root, List *clauses, Assert(list_length(bool_clauses) >= 2); /* build the match bitmap for the OR-clauses */ - bool_matches = mcv_get_match_bitmap(root, bool_clauses, keys, exprs, + bool_matches = mcv_get_match_bitmap(root, bool_clauses, exprs, mcvlist, is_orclause(clause)); /* @@ -1891,7 +1880,7 @@ mcv_get_match_bitmap(PlannerInfo *root, List *clauses, Assert(list_length(not_args) == 1); /* build the match bitmap for the NOT-clause */ - not_matches = mcv_get_match_bitmap(root, not_args, keys, exprs, + not_matches = mcv_get_match_bitmap(root, not_args, exprs, mcvlist, false); /* @@ -1911,7 +1900,7 @@ mcv_get_match_bitmap(PlannerInfo *root, List *clauses, Var *var = (Var *) (clause); /* match the attribute to a dimension of the statistic */ - int idx = bms_member_index(keys, var->varattno); + int idx = mcv_match_expression((Node *) var, exprs, NULL); Assert(var->vartype == BOOLOID); @@ -1939,7 +1928,7 @@ mcv_get_match_bitmap(PlannerInfo *root, List *clauses, int idx; /* match the expression to a dimension of the statistic */ - idx = mcv_match_expression(clause, keys, exprs, NULL); + idx = mcv_match_expression(clause, exprs, NULL); /* * Walk through the MCV items and evaluate the current clause. We @@ -2057,7 +2046,7 @@ mcv_clauselist_selectivity(PlannerInfo *root, StatisticExtInfo *stat, mcv = statext_mcv_load(stat->statOid, rte->inh); /* build a match bitmap for the clauses */ - matches = mcv_get_match_bitmap(root, clauses, stat->keys, stat->exprs, + matches = mcv_get_match_bitmap(root, clauses, stat->exprs, mcv, false); /* sum frequencies for all the matching MCV items */ @@ -2130,8 +2119,8 @@ mcv_clause_selectivity_or(PlannerInfo *root, StatisticExtInfo *stat, *or_matches = palloc0_array(bool, mcv->nitems); /* build the match bitmap for the new clause */ - new_matches = mcv_get_match_bitmap(root, list_make1(clause), stat->keys, - stat->exprs, mcv, false); + new_matches = mcv_get_match_bitmap(root, list_make1(clause), stat->exprs, + mcv, false); /* * Sum the frequencies for all the MCV items matching this clause and also diff --git a/src/backend/statistics/mvdistinct.c b/src/backend/statistics/mvdistinct.c index a6a904039e5..47f4c614285 100644 --- a/src/backend/statistics/mvdistinct.c +++ b/src/backend/statistics/mvdistinct.c @@ -121,6 +121,12 @@ statext_ndistinct_build(double totalrows, StatsBuildData *data) Assert(AttributeNumberIsValid(item->attributes[j])); } + /* + * Order the attnums the way dump and restore needs (see + * compare_attnums). + */ + qsort(item->attributes, k, sizeof(AttrNumber), compare_attnums); + item->ndistinct = ndistinct_for_combination(totalrows, data, k, combination); @@ -342,13 +348,13 @@ statext_ndistinct_free(MVNDistinct *ndistinct) * attributes list correspond to attnums/expressions defined by the extended * statistics object. * - * Positive attnums are attributes which must be found in the stxkeys, - * while negative attnums correspond to an expression number, no attribute + * Positive attnums correspond to table columns (excluding virtual generated + * columns), while negative attnums correspond to expressions. No attribute * number can be below (0 - numexprs). */ bool statext_ndistinct_validate(const MVNDistinct *ndistinct, - const int2vector *stxkeys, + const Bitmapset *keys, int numexprs, int elevel) { int attnum_expr_lowbound = 0 - numexprs; @@ -369,15 +375,8 @@ statext_ndistinct_validate(const MVNDistinct *ndistinct, if (attnum > 0) { - /* attribute number in stxkeys */ - for (int k = 0; k < stxkeys->dim1; k++) - { - if (attnum == stxkeys->values[k]) - { - ok = true; - break; - } - } + /* attribute number in keys */ + ok = bms_is_member(attnum, keys); } else if ((attnum < 0) && (attnum >= attnum_expr_lowbound)) { diff --git a/src/backend/utils/adt/ruleutils.c b/src/backend/utils/adt/ruleutils.c index 1d41c06201b..d8a4a7ca160 100644 --- a/src/backend/utils/adt/ruleutils.c +++ b/src/backend/utils/adt/ruleutils.c @@ -62,6 +62,7 @@ #include "rewrite/rewriteHandler.h" #include "rewrite/rewriteManip.h" #include "rewrite/rewriteSupport.h" +#include "statistics/statistics.h" #include "utils/array.h" #include "utils/builtins.h" #include "utils/fmgroids.h" @@ -369,7 +370,7 @@ static char *pg_get_indexdef_worker(Oid indexrelid, int colno, static void make_propgraphdef_elements(StringInfo buf, Oid pgrelid, char pgekind); static void make_propgraphdef_labels(StringInfo buf, Oid elid, const char *elalias, Oid elrelid); static void make_propgraphdef_properties(StringInfo buf, Oid ellabelid, Oid elrelid); -static char *pg_get_statisticsobj_worker(Oid statextid, bool columns_only, +static char *pg_get_statisticsobj_worker(Oid statextid, bool missing_ok); static char *pg_get_partkeydef_worker(Oid relid, int prettyFlags, bool attrsOnly, bool missing_ok); @@ -1972,7 +1973,7 @@ pg_get_statisticsobjdef(PG_FUNCTION_ARGS) Oid statextid = PG_GETARG_OID(0); char *res; - res = pg_get_statisticsobj_worker(statextid, false, true); + res = pg_get_statisticsobj_worker(statextid, true); if (res == NULL) PG_RETURN_NULL(); @@ -1987,7 +1988,27 @@ pg_get_statisticsobjdef(PG_FUNCTION_ARGS) char * pg_get_statisticsobjdef_string(Oid statextid) { - return pg_get_statisticsobj_worker(statextid, false, false); + return pg_get_statisticsobj_worker(statextid, false); +} + +/* + * Deparse one column or expression of an extended statistics object into the + * text used in its CREATE STATISTICS command. + */ +static char * +deparse_stat_entry(Node *expr, Oid relid, List *context) +{ + char *str; + + if (IsA(expr, Var) && ((Var *) expr)->varattno > 0) + return pstrdup(quote_identifier( + get_attname(relid, ((Var *) expr)->varattno, false))); + + str = deparse_expression_pretty(expr, context, false, false, + PRETTYFLAG_PAREN, 0); + if (looks_like_function(expr)) + return str; + return psprintf("(%s)", str); } /* @@ -1998,21 +2019,57 @@ Datum pg_get_statisticsobjdef_columns(PG_FUNCTION_ARGS) { Oid statextid = PG_GETARG_OID(0); - char *res; + Form_pg_statistic_ext statextrec; + HeapTuple statexttup; + Datum datum; + List *allexprs = NIL; + char *tmp; + List *context; + ListCell *lc; + ArrayBuildState *astate = NULL; - res = pg_get_statisticsobj_worker(statextid, true, true); + statexttup = SearchSysCache1(STATEXTOID, ObjectIdGetDatum(statextid)); - if (res == NULL) + if (!HeapTupleIsValid(statexttup)) PG_RETURN_NULL(); - PG_RETURN_TEXT_P(string_to_text(res)); + statextrec = (Form_pg_statistic_ext) GETSTRUCT(statexttup); + + datum = SysCacheGetAttrNotNull(STATEXTOID, statexttup, + Anum_pg_statistic_ext_stxexprs); + tmp = TextDatumGetCString(datum); + allexprs = (List *) stringToNode(tmp); + pfree(tmp); + + context = deparse_context_for(get_relation_name(statextrec->stxrelid), + statextrec->stxrelid); + + foreach(lc, allexprs) + { + Node *expr = (Node *) lfirst(lc); + char *str; + + str = deparse_stat_entry(expr, statextrec->stxrelid, context); + astate = accumArrayResult(astate, + PointerGetDatum(cstring_to_text(str)), + false, + TEXTOID, + CurrentMemoryContext); + } + + ReleaseSysCache(statexttup); + + if (astate == NULL) + PG_RETURN_NULL(); + + PG_RETURN_DATUM(makeArrayResult(astate, CurrentMemoryContext)); } /* * Internal workhorse to decompile an extended statistics object. */ static char * -pg_get_statisticsobj_worker(Oid statextid, bool columns_only, bool missing_ok) +pg_get_statisticsobj_worker(Oid statextid, bool missing_ok) { Form_pg_statistic_ext statextrec; HeapTuple statexttup; @@ -2022,6 +2079,7 @@ pg_get_statisticsobj_worker(Oid statextid, bool columns_only, bool missing_ok) ArrayType *arr; char *enabled; Datum datum; + char *exprsString; bool ndistinct_enabled; bool dependencies_enabled; bool mcv_enabled; @@ -2029,7 +2087,6 @@ pg_get_statisticsobj_worker(Oid statextid, bool columns_only, bool missing_ok) List *context; ListCell *lc; List *exprs = NIL; - bool has_exprs; int ncolumns; statexttup = SearchSysCache1(STATEXTOID, ObjectIdGetDatum(statextid)); @@ -2041,151 +2098,113 @@ pg_get_statisticsobj_worker(Oid statextid, bool columns_only, bool missing_ok) elog(ERROR, "cache lookup failed for statistics object %u", statextid); } - /* has the statistics expressions? */ - has_exprs = !heap_attisnull(statexttup, Anum_pg_statistic_ext_stxexprs, NULL); - statextrec = (Form_pg_statistic_ext) GETSTRUCT(statexttup); /* - * Get the statistics expressions, if any. (NOTE: we do not use the - * relcache versions of the expressions, because we want to display + * Get the statistics expressions. (NOTE: we do not use the relcache + * versions of the expressions, because we want to display * non-const-folded expressions.) */ - if (has_exprs) - { - Datum exprsDatum; - char *exprsString; - - exprsDatum = SysCacheGetAttrNotNull(STATEXTOID, statexttup, - Anum_pg_statistic_ext_stxexprs); - exprsString = TextDatumGetCString(exprsDatum); - exprs = (List *) stringToNode(exprsString); - pfree(exprsString); - } - else - exprs = NIL; + datum = SysCacheGetAttrNotNull(STATEXTOID, statexttup, + Anum_pg_statistic_ext_stxexprs); + exprsString = TextDatumGetCString(datum); + exprs = (List *) stringToNode(exprsString); + pfree(exprsString); /* count the number of columns (attributes and expressions) */ - ncolumns = statextrec->stxkeys.dim1 + list_length(exprs); + ncolumns = list_length(exprs); initStringInfo(&buf); - if (!columns_only) - { - nsp = get_namespace_name_or_temp(statextrec->stxnamespace); - appendStringInfo(&buf, "CREATE STATISTICS %s", - quote_qualified_identifier(nsp, - NameStr(statextrec->stxname))); + nsp = get_namespace_name_or_temp(statextrec->stxnamespace); + appendStringInfo(&buf, "CREATE STATISTICS %s", + quote_qualified_identifier(nsp, + NameStr(statextrec->stxname))); - /* - * Decode the stxkind column so that we know which stats types to - * print. - */ - datum = SysCacheGetAttrNotNull(STATEXTOID, statexttup, - Anum_pg_statistic_ext_stxkind); - arr = DatumGetArrayTypeP(datum); - if (ARR_NDIM(arr) != 1 || - ARR_HASNULL(arr) || - ARR_ELEMTYPE(arr) != CHAROID) - elog(ERROR, "stxkind is not a 1-D char array"); - enabled = (char *) ARR_DATA_PTR(arr); - - ndistinct_enabled = false; - dependencies_enabled = false; - mcv_enabled = false; - - for (i = 0; i < ARR_DIMS(arr)[0]; i++) - { - if (enabled[i] == STATS_EXT_NDISTINCT) - ndistinct_enabled = true; - else if (enabled[i] == STATS_EXT_DEPENDENCIES) - dependencies_enabled = true; - else if (enabled[i] == STATS_EXT_MCV) - mcv_enabled = true; - - /* ignore STATS_EXT_EXPRESSIONS (it's built automatically) */ - } + /* + * Decode the stxkind column so that we know which stats types to print. + */ + datum = SysCacheGetAttrNotNull(STATEXTOID, statexttup, + Anum_pg_statistic_ext_stxkind); + arr = DatumGetArrayTypeP(datum); + if (ARR_NDIM(arr) != 1 || + ARR_HASNULL(arr) || + ARR_ELEMTYPE(arr) != CHAROID) + elog(ERROR, "stxkind is not a 1-D char array"); + enabled = (char *) ARR_DATA_PTR(arr); - /* - * If any option is disabled, then we'll need to append the types - * clause to show which options are enabled. We omit the types clause - * on purpose when all options are enabled, so a pg_dump/pg_restore - * will create all statistics types on a newer postgres version, if - * the statistics had all options enabled on the original version. - * - * But if the statistics is defined on just a single column, it has to - * be an expression statistics. In that case we don't need to specify - * kinds. - */ - if ((!ndistinct_enabled || !dependencies_enabled || !mcv_enabled) && - (ncolumns > 1)) - { - bool gotone = false; + ndistinct_enabled = false; + dependencies_enabled = false; + mcv_enabled = false; - appendStringInfoString(&buf, " ("); + for (i = 0; i < ARR_DIMS(arr)[0]; i++) + { + if (enabled[i] == STATS_EXT_NDISTINCT) + ndistinct_enabled = true; + else if (enabled[i] == STATS_EXT_DEPENDENCIES) + dependencies_enabled = true; + else if (enabled[i] == STATS_EXT_MCV) + mcv_enabled = true; - if (ndistinct_enabled) - { - appendStringInfoString(&buf, "ndistinct"); - gotone = true; - } + /* ignore STATS_EXT_EXPRESSIONS (it's built automatically) */ + } - if (dependencies_enabled) - { - appendStringInfo(&buf, "%sdependencies", gotone ? ", " : ""); - gotone = true; - } + /* + * If any option is disabled, then we'll need to append the types clause + * to show which options are enabled. We omit the types clause on purpose + * when all options are enabled, so a pg_dump/pg_restore will create all + * statistics types on a newer postgres version, if the statistics had all + * options enabled on the original version. + * + * But if the statistics is defined on just a single column, it has to be + * an expression statistics. In that case we don't need to specify kinds. + */ + if ((!ndistinct_enabled || !dependencies_enabled || !mcv_enabled) && + (ncolumns > 1)) + { + bool gotone = false; - if (mcv_enabled) - appendStringInfo(&buf, "%smcv", gotone ? ", " : ""); + appendStringInfoString(&buf, " ("); - appendStringInfoChar(&buf, ')'); + if (ndistinct_enabled) + { + appendStringInfoString(&buf, "ndistinct"); + gotone = true; } - appendStringInfoString(&buf, " ON "); - } - - /* decode simple column references */ - for (colno = 0; colno < statextrec->stxkeys.dim1; colno++) - { - AttrNumber attnum = statextrec->stxkeys.values[colno]; - char *attname; - - if (colno > 0) - appendStringInfoString(&buf, ", "); + if (dependencies_enabled) + { + appendStringInfo(&buf, "%sdependencies", gotone ? ", " : ""); + gotone = true; + } - attname = get_attname(statextrec->stxrelid, attnum, false); + if (mcv_enabled) + appendStringInfo(&buf, "%smcv", gotone ? ", " : ""); - appendStringInfoString(&buf, quote_identifier(attname)); + appendStringInfoChar(&buf, ')'); } + appendStringInfoString(&buf, " ON "); + context = deparse_context_for(get_relation_name(statextrec->stxrelid), statextrec->stxrelid); + colno = 0; foreach(lc, exprs) { Node *expr = (Node *) lfirst(lc); - char *str; - int prettyFlags = PRETTYFLAG_PAREN; - - str = deparse_expression_pretty(expr, context, false, false, - prettyFlags, 0); if (colno > 0) appendStringInfoString(&buf, ", "); - /* Need parens if it's not a bare function call */ - if (looks_like_function(expr)) - appendStringInfoString(&buf, str); - else - appendStringInfo(&buf, "(%s)", str); - + appendStringInfoString(&buf, + deparse_stat_entry(expr, statextrec->stxrelid, + context)); colno++; } - if (!columns_only) - appendStringInfo(&buf, " FROM %s", - generate_relation_name(statextrec->stxrelid, NIL)); + appendStringInfo(&buf, " FROM %s", + generate_relation_name(statextrec->stxrelid, NIL)); ReleaseSysCache(statexttup); @@ -2202,10 +2221,12 @@ pg_get_statisticsobjdef_expressions(PG_FUNCTION_ARGS) Form_pg_statistic_ext statextrec; HeapTuple statexttup; Datum datum; + Relation rel; List *context; ListCell *lc; + ListCell *lc_raw; + List *rawexprs = NIL; List *exprs = NIL; - bool has_exprs; char *tmp; ArrayBuildState *astate = NULL; @@ -2214,37 +2235,51 @@ pg_get_statisticsobjdef_expressions(PG_FUNCTION_ARGS) if (!HeapTupleIsValid(statexttup)) PG_RETURN_NULL(); - /* Does the stats object have expressions? */ - has_exprs = !heap_attisnull(statexttup, Anum_pg_statistic_ext_stxexprs, NULL); - - /* no expressions? we're done */ - if (!has_exprs) - { - ReleaseSysCache(statexttup); - PG_RETURN_NULL(); - } - statextrec = (Form_pg_statistic_ext) GETSTRUCT(statexttup); /* - * Get the statistics expressions, and deparse them into text values. + * Decode stxexprs as originally written; we deparse these, because we + * want to display the expressions without const-folding. */ datum = SysCacheGetAttrNotNull(STATEXTOID, statexttup, Anum_pg_statistic_ext_stxexprs); tmp = TextDatumGetCString(datum); - exprs = (List *) stringToNode(tmp); + rawexprs = (List *) stringToNode(tmp); pfree(tmp); + /* + * exprs is rawexprs with virtual generated columns expanded and + * expressions const-folded; the transforms are per-node, so it stays + * element-aligned with rawexprs -- we classify on the folded exprs but + * deparse the original rawexpr. Dropping the plain-column entries leaves + * exactly the complex expressions, in the same order as stxdexpr, which + * pg_stats_ext_exprs pairs this output with. + */ + rel = table_open(statextrec->stxrelid, AccessShareLock); + exprs = statext_get_stxexprs(statexttup, rel); + table_close(rel, AccessShareLock); + + Assert(list_length(exprs) == list_length(rawexprs)); + context = deparse_context_for(get_relation_name(statextrec->stxrelid), statextrec->stxrelid); - foreach(lc, exprs) + forboth(lc, exprs, lc_raw, rawexprs) { Node *expr = (Node *) lfirst(lc); + Node *rawexpr = (Node *) lfirst(lc_raw); char *str; int prettyFlags = PRETTYFLAG_INDENT; - str = deparse_expression_pretty(expr, context, false, false, + /* + * Skip entries that resolve to a plain column: their statistics are + * in pg_statistic, not in pg_statistic_ext_data.stxdexpr, so they are + * not part of the list we return. + */ + if (statext_is_column(expr)) + continue; + + str = deparse_expression_pretty(rawexpr, context, false, false, prettyFlags, 0); astate = accumArrayResult(astate, @@ -2256,6 +2291,9 @@ pg_get_statisticsobjdef_expressions(PG_FUNCTION_ARGS) ReleaseSysCache(statexttup); + if (astate == NULL) + PG_RETURN_NULL(); + PG_RETURN_DATUM(makeArrayResult(astate, CurrentMemoryContext)); } diff --git a/src/backend/utils/adt/selfuncs.c b/src/backend/utils/adt/selfuncs.c index 2b4e6acf9a7..50e1413870f 100644 --- a/src/backend/utils/adt/selfuncs.c +++ b/src/backend/utils/adt/selfuncs.c @@ -4627,7 +4627,7 @@ estimate_multivariate_ndistinct(PlannerInfo *root, RelOptInfo *rel, if (!AttrNumberIsForUserDefinedAttr(attnum)) continue; - if (bms_is_member(attnum, info->keys)) + if (stat_covers_attnum(info, attnum)) nshared_vars++; continue; @@ -4692,14 +4692,15 @@ estimate_multivariate_ndistinct(PlannerInfo *root, RelOptInfo *rel, ListCell *lc2; Bitmapset *matched = NULL; AttrNumber attnum_offset; + int nexprs = stat_num_expressions(matched_info); /* * How much we need to offset the attnums? If there are no * expressions, no offset is needed. Otherwise offset enough to move * the lowest one (which is equal to number of expressions) to 1. */ - if (matched_info->exprs) - attnum_offset = (list_length(matched_info->exprs) + 1); + if (nexprs > 0) + attnum_offset = nexprs + 1; else attnum_offset = 0; @@ -4729,7 +4730,7 @@ estimate_multivariate_ndistinct(PlannerInfo *root, RelOptInfo *rel, continue; /* Is the variable covered by the statistics object? */ - if (!bms_is_member(attnum, matched_info->keys)) + if (!stat_covers_attnum(matched_info, attnum)) continue; attnum = attnum + attnum_offset; @@ -4756,6 +4757,13 @@ estimate_multivariate_ndistinct(PlannerInfo *root, RelOptInfo *rel, { Node *expr = (Node *) lfirst(lc3); + /* + * columns have no per-expression stats, so count expressions + * only + */ + if (statext_is_column(expr)) + continue; + if (equal(varinfo->var, expr)) { AttrNumber attnum = -(idx + 1); @@ -5922,6 +5930,13 @@ examine_variable(PlannerInfo *root, Node *node, int varRelid, { Node *expr = (Node *) lfirst(expr_item); + /* + * columns have no per-expression stats, so count expressions + * only + */ + if (statext_is_column(expr)) + continue; + Assert(expr); /* strip RelabelType before comparing it */ diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c index ad9c8affb4f..6525a4a3fc5 100644 --- a/src/bin/psql/describe.c +++ b/src/bin/psql/describe.c @@ -2767,8 +2767,15 @@ describeOneTableDetails(const char *schemaname, "SELECT oid, " "stxrelid::pg_catalog.regclass, " "stxnamespace::pg_catalog.regnamespace::pg_catalog.text AS nsp, " - "stxname,\n" - "pg_catalog.pg_get_statisticsobjdef_columns(oid) AS columns,\n" + "stxname,\n"); + /* TODO: update threshold to 200000 when PG20 version is assigned */ + if (pset.sversion >= 190000) + appendPQExpBufferStr(&buf, + "pg_catalog.array_to_string(pg_catalog.pg_get_statisticsobjdef_columns(oid), ', ') AS columns,\n"); + else + appendPQExpBufferStr(&buf, + "pg_catalog.pg_get_statisticsobjdef_columns(oid) AS columns,\n"); + appendPQExpBuffer(&buf, " " CppAsString2(STATS_EXT_NDISTINCT) " = any(stxkind) AS ndist_enabled,\n" " " CppAsString2(STATS_EXT_DEPENDENCIES) " = any(stxkind) AS deps_enabled,\n" " " CppAsString2(STATS_EXT_MCV) " = any(stxkind) AS mcv_enabled,\n" @@ -4946,7 +4953,14 @@ listExtendedStats(const char *pattern, bool verbose) gettext_noop("Schema"), gettext_noop("Name")); - if (pset.sversion >= 140000) + /* TODO: update threshold to 200000 when PG20 version is assigned */ + if (pset.sversion >= 190000) + appendPQExpBuffer(&buf, + "pg_catalog.format('%%s FROM %%s', \n" + " pg_catalog.array_to_string(pg_catalog.pg_get_statisticsobjdef_columns(es.oid), ', '), \n" + " es.stxrelid::pg_catalog.regclass) AS \"%s\"", + gettext_noop("Definition")); + else if (pset.sversion >= 140000) appendPQExpBuffer(&buf, "pg_catalog.format('%%s FROM %%s', \n" " pg_catalog.pg_get_statisticsobjdef_columns(es.oid), \n" diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index f8a021987b5..aece868984c 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -3998,9 +3998,9 @@ proname => 'pg_get_statisticsobjdef', provolatile => 's', prorettype => 'text', proargtypes => 'oid', prosrc => 'pg_get_statisticsobjdef' }, -{ oid => '6174', descr => 'extended statistics columns', +{ oid => '6174', descr => 'extended statistics columns and expressions', proname => 'pg_get_statisticsobjdef_columns', provolatile => 's', - prorettype => 'text', proargtypes => 'oid', + prorettype => '_text', proargtypes => 'oid', prosrc => 'pg_get_statisticsobjdef_columns' }, { oid => '6173', descr => 'extended statistics expressions', proname => 'pg_get_statisticsobjdef_expressions', provolatile => 's', diff --git a/src/include/catalog/pg_statistic_ext.h b/src/include/catalog/pg_statistic_ext.h index e4a0cb4d41c..9cfb39f7f3d 100644 --- a/src/include/catalog/pg_statistic_ext.h +++ b/src/include/catalog/pg_statistic_ext.h @@ -46,19 +46,15 @@ CATALOG(pg_statistic_ext,3381,StatisticExtRelationId) Oid stxowner BKI_LOOKUP(pg_authid); /* statistics object's owner */ - /* - * variable-length/nullable fields start here, but we allow direct access - * to stxkeys - */ - int2vector stxkeys BKI_FORCE_NOT_NULL; /* array of column keys */ - #ifdef CATALOG_VARLEN int16 stxstattarget BKI_DEFAULT(_null_) BKI_FORCE_NULL; /* statistics target */ char stxkind[1] BKI_FORCE_NOT_NULL; /* statistics kinds requested * to build */ - pg_node_tree stxexprs; /* A list of expression trees for stats - * attributes that are not simple column - * references. */ + pg_node_tree stxexprs BKI_FORCE_NOT_NULL; /* expression trees for the + * columns and expressions the + * statistics object is + * defined on; simple columns + * are Var nodes */ #endif } FormData_pg_statistic_ext; @@ -81,7 +77,6 @@ DECLARE_INDEX(pg_statistic_ext_relid_index, 3379, StatisticExtRelidIndexId, pg_s MAKE_SYSCACHE(STATEXTOID, pg_statistic_ext_oid_index, 4); MAKE_SYSCACHE(STATEXTNAMENSP, pg_statistic_ext_name_index, 4); -DECLARE_ARRAY_FOREIGN_KEY((stxrelid, stxkeys), pg_attribute, (attrelid, attnum)); #ifdef EXPOSE_TO_CLIENT_CODE diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index 27a2c6815b7..557b7337ea8 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -1525,10 +1525,7 @@ typedef struct StatisticExtInfo /* statistics kind of this entry */ char kind; - /* attnums of the columns covered */ - Bitmapset *keys; - - /* expressions */ + /* all columns and expressions the object is defined on, in declared order */ List *exprs; } StatisticExtInfo; diff --git a/src/include/statistics/extended_stats_internal.h b/src/include/statistics/extended_stats_internal.h index c775442f2ee..8e18652659b 100644 --- a/src/include/statistics/extended_stats_internal.h +++ b/src/include/statistics/extended_stats_internal.h @@ -73,7 +73,7 @@ extern MVNDistinct *statext_ndistinct_build(double totalrows, StatsBuildData *da extern bytea *statext_ndistinct_serialize(MVNDistinct *ndistinct); extern MVNDistinct *statext_ndistinct_deserialize(bytea *data); extern bool statext_ndistinct_validate(const MVNDistinct *ndistinct, - const int2vector *stxkeys, + const Bitmapset *keys, int numexprs, int elevel); extern void statext_ndistinct_free(MVNDistinct *ndistinct); @@ -81,7 +81,7 @@ extern MVDependencies *statext_dependencies_build(StatsBuildData *data); extern bytea *statext_dependencies_serialize(MVDependencies *dependencies); extern MVDependencies *statext_dependencies_deserialize(bytea *data); extern bool statext_dependencies_validate(const MVDependencies *dependencies, - const int2vector *stxkeys, + const Bitmapset *keys, int numexprs, int elevel); extern void statext_dependencies_free(MVDependencies *dependencies); @@ -106,8 +106,7 @@ extern int multi_sort_compare_dims(int start, int end, const SortItem *a, const SortItem *b, MultiSortSupport mss); extern int compare_scalars_simple(const void *a, const void *b, void *arg); extern int compare_datums_simple(Datum a, Datum b, SortSupport ssup); - -extern AttrNumber *build_attnums_array(Bitmapset *attrs, int nexprs, int *numattrs); +extern int compare_attnums(const void *a, const void *b); extern SortItem *build_sorted_items(StatsBuildData *data, int *nitems, MultiSortSupport mss, diff --git a/src/include/statistics/statistics.h b/src/include/statistics/statistics.h index 0b163103a72..772abfc3579 100644 --- a/src/include/statistics/statistics.h +++ b/src/include/statistics/statistics.h @@ -127,6 +127,11 @@ extern StatisticExtInfo *choose_best_statistics(List *stats, char requiredkind, List **clause_exprs, int nclauses); extern HeapTuple statext_expressions_load(Oid stxoid, bool inh, int idx); +extern List *statext_get_stxexprs(HeapTuple htup, Relation rel); +extern bool statext_is_column(Node *node); +extern bool stat_covers_attnum(StatisticExtInfo *stat, AttrNumber attnum); +extern int stat_num_expressions(StatisticExtInfo *stat); +extern Node *stat_nth_expression(StatisticExtInfo *stat, int n); extern bool import_relation_statistics(Relation rel, const NullableDatum *version, diff --git a/src/test/regress/expected/create_table_like.out b/src/test/regress/expected/create_table_like.out index a23735b5fb4..dcc26b2eb67 100644 --- a/src/test/regress/expected/create_table_like.out +++ b/src/test/regress/expected/create_table_like.out @@ -699,7 +699,7 @@ SELECT attname, attcompression FROM pg_attribute (5 rows) -- LIKE ... INCLUDING STATISTICS with dropped columns in the parent, --- so stxkeys attnums are not contiguous. +-- so column attnums are not contiguous. CREATE TABLE ctl_stats3_parent (a int, b int, c int); ALTER TABLE ctl_stats3_parent DROP COLUMN b; CREATE STATISTICS ctl_stats3_stat ON a, c FROM ctl_stats3_parent; @@ -709,15 +709,10 @@ ALTER TABLE ctl_stats4_parent DROP COLUMN b; CREATE STATISTICS ctl_stats4_stat ON a, c FROM ctl_stats4_parent; CREATE TABLE ctl_stats4_child (LIKE ctl_stats4_parent INCLUDING STATISTICS); SELECT s.stxrelid::regclass AS relation, - array_agg(a.attname ORDER BY u.ord) AS stats_columns + pg_get_statisticsobjdef_columns(s.oid) AS stats_columns FROM pg_statistic_ext s -CROSS JOIN LATERAL - unnest(s.stxkeys::int2[]) WITH ORDINALITY AS u(attnum, ord) -JOIN pg_attribute a - ON a.attrelid = s.stxrelid AND a.attnum = u.attnum WHERE s.stxrelid IN ('ctl_stats3_child'::regclass, 'ctl_stats4_child'::regclass) -GROUP BY s.stxrelid ORDER BY s.stxrelid::regclass::text; relation | stats_columns ------------------+--------------- diff --git a/src/test/regress/expected/oidjoins.out b/src/test/regress/expected/oidjoins.out index d64169b7bf0..0e6bce84a60 100644 --- a/src/test/regress/expected/oidjoins.out +++ b/src/test/regress/expected/oidjoins.out @@ -166,7 +166,6 @@ NOTICE: checking pg_statistic {starelid,staattnum} => pg_attribute {attrelid,at NOTICE: checking pg_statistic_ext {stxrelid} => pg_class {oid} NOTICE: checking pg_statistic_ext {stxnamespace} => pg_namespace {oid} NOTICE: checking pg_statistic_ext {stxowner} => pg_authid {oid} -NOTICE: checking pg_statistic_ext {stxrelid,stxkeys} => pg_attribute {attrelid,attnum} NOTICE: checking pg_statistic_ext_data {stxoid} => pg_statistic_ext {oid} NOTICE: checking pg_rewrite {ev_class} => pg_class {oid} NOTICE: checking pg_trigger {tgrelid} => pg_class {oid} diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 6a3341356da..395eb0b5f8b 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -2715,10 +2715,7 @@ pg_stats_ext| SELECT cn.nspname AS schemaname, s.stxname AS statistics_name, s.oid AS statistics_id, pg_get_userbyid(s.stxowner) AS statistics_owner, - ( SELECT array_agg(a.attname ORDER BY a.attnum) AS array_agg - FROM (unnest(s.stxkeys) k(k) - JOIN pg_attribute a ON (((a.attrelid = s.stxrelid) AND (a.attnum = k.k))))) AS attnames, - pg_get_statisticsobjdef_expressions(s.oid) AS exprs, + pg_get_statisticsobjdef_columns(s.oid) AS exprs, s.stxkind AS kinds, sd.stxdinherit AS inherited, sd.stxdndistinct AS n_distinct, diff --git a/src/test/regress/expected/stats_ext.out b/src/test/regress/expected/stats_ext.out index 37070c1a896..082f490a2ce 100644 --- a/src/test/regress/expected/stats_ext.out +++ b/src/test/regress/expected/stats_ext.out @@ -43,7 +43,7 @@ ERROR: relation "nonexistent" does not exist CREATE STATISTICS tst ON a, b FROM ext_stats_test; ERROR: column "a" does not exist CREATE STATISTICS tst ON x, x, y FROM ext_stats_test; -ERROR: duplicate column name in statistics definition +ERROR: duplicate column or expression in statistics definition CREATE STATISTICS tst ON x, x, y, x, x, y, x, x, y FROM ext_stats_test; ERROR: cannot have more than 8 columns in statistics CREATE STATISTICS tst ON x, x, y, x, x, (x || 'x'), (y + 1), (x || 'x'), (x || 'x'), (y + 1) FROM ext_stats_test; @@ -51,7 +51,7 @@ ERROR: cannot have more than 8 columns in statistics CREATE STATISTICS tst ON (x || 'x'), (x || 'x'), (y + 1), (x || 'x'), (x || 'x'), (y + 1), (x || 'x'), (x || 'x'), (y + 1) FROM ext_stats_test; ERROR: cannot have more than 8 columns in statistics CREATE STATISTICS tst ON (x || 'x'), (x || 'x'), y FROM ext_stats_test; -ERROR: duplicate expression in statistics definition +ERROR: duplicate column or expression in statistics definition CREATE STATISTICS tst (unrecognized) ON x, y FROM ext_stats_test; ERROR: unrecognized statistics kind "unrecognized" -- unsupported targets @@ -306,6 +306,58 @@ SELECT * FROM check_estimated_rows('SELECT a + 1, b FROM ONLY stxdinp GROUP BY 1 (1 row) DROP TABLE stxdinp; +-- a column-equivalent expression (one that const-folds to a plain column, or a +-- passthrough virtual generated column) is treated as that column +CREATE TABLE column_equivalent (a int, b int, g int GENERATED ALWAYS AS (a) VIRTUAL) WITH (autovacuum_enabled = off); +INSERT INTO column_equivalent (a, b) SELECT i%100, i%10 FROM generate_series(1, 10000) i; +-- a column-equivalent CASE between two complex expressions +CREATE STATISTICS column_equivalent_middle ON (a + b), (CASE WHEN true THEN a ELSE b END), (a * b) FROM column_equivalent; +-- only column-equivalent entries, which all fold to plain columns +CREATE STATISTICS column_equivalent_all ON (CASE WHEN true THEN a END), b FROM column_equivalent; +ANALYZE column_equivalent; +-- a column-equivalent CASE is not reported as an expression; only the complex +-- expressions appear, each with its own statistics +SELECT statistics_name, expr, n_distinct +FROM pg_stats_ext_exprs +WHERE statistics_name IN ('column_equivalent_middle', 'column_equivalent_all') +ORDER BY statistics_name, expr; + statistics_name | expr | n_distinct +--------------------------+---------+------------ + column_equivalent_middle | (a * b) | 84 + column_equivalent_middle | (a + b) | 55 +(2 rows) + +-- each complex expression's GROUP BY estimate uses its own statistics +SELECT * FROM check_estimated_rows('SELECT (a + b) FROM column_equivalent GROUP BY 1'); + estimated | actual +-----------+-------- + 55 | 55 +(1 row) + +SELECT * FROM check_estimated_rows('SELECT (a * b) FROM column_equivalent GROUP BY 1'); + estimated | actual +-----------+-------- + 84 | 84 +(1 row) + +-- column_equivalent_all is used for a GROUP BY on the plain columns a, b +SELECT * FROM check_estimated_rows('SELECT COUNT(*) FROM column_equivalent GROUP BY a, b'); + estimated | actual +-----------+-------- + 100 | 100 +(1 row) + +-- an entry that reduces to a column already listed is rejected as a duplicate: +-- a parenthesized column, +CREATE STATISTICS column_equivalent_dup_paren ON a, (a) FROM column_equivalent; +ERROR: duplicate column or expression in statistics definition +-- a constant-folding expression, +CREATE STATISTICS column_equivalent_dup_case ON a, (CASE WHEN true THEN a END), b FROM column_equivalent; +ERROR: duplicate column or expression in statistics definition +-- and a passthrough generated column +CREATE STATISTICS column_equivalent_dup_vgen ON a, g FROM column_equivalent; +ERROR: duplicate column or expression in statistics definition +DROP TABLE column_equivalent; -- basic test for statistics on expressions CREATE TABLE ab1 (a INTEGER, b INTEGER, c TIMESTAMP, d TIMESTAMPTZ); -- expression stats may be built on a single expression column @@ -3177,6 +3229,23 @@ SELECT * FROM check_estimated_rows('SELECT * FROM virtual_gen_stats WHERE d = 0 10 | 10 (1 row) +SELECT expr FROM pg_stats_ext_exprs + WHERE statistics_name = 'virtual_gen_stats_1' AND NOT inherited; + expr +--------------- + c + (3 * b) + d + (d - (2 * a)) +(4 rows) + +SELECT pg_get_statisticsobjdef(oid) FROM pg_statistic_ext + WHERE stxname = 'virtual_gen_stats_1'; + pg_get_statisticsobjdef +--------------------------------------------------------------------------------------------------------- + CREATE STATISTICS public.virtual_gen_stats_1 (mcv) ON c, (3 * b), d, (d - 2 * a) FROM virtual_gen_stats +(1 row) + -- univariate statistics on individual virtual generated columns DROP STATISTICS virtual_gen_stats_1; SELECT * FROM check_estimated_rows('SELECT * FROM virtual_gen_stats WHERE c = 0'); @@ -3692,12 +3761,12 @@ CREATE STATISTICS stats_ext_range (mcv) ON irange, (irange + '[4,10)'::int4range) FROM stats_ext_tbl_range; ANALYZE stats_ext_tbl_range; -SELECT attnames, most_common_vals +SELECT exprs, most_common_vals FROM pg_stats_ext WHERE statistics_name = 'stats_ext_range'; - attnames | most_common_vals -----------+------------------------------------------------------------ - {irange} | {{"[1,7)","[1,10)"},{"[2,9)","[2,10)"},{"[3,9)","[3,10)"}} + exprs | most_common_vals +-------------------------------------------+------------------------------------------------------------ + {irange,"(irange + '[4,10)'::int4range)"} | {{"[1,7)","[1,10)"},{"[2,9)","[2,10)"},{"[3,9)","[3,10)"}} (1 row) SELECT range_length_histogram, range_empty_frac, range_bounds_histogram @@ -3709,3 +3778,65 @@ SELECT range_length_histogram, range_empty_frac, range_bounds_histogram (1 row) DROP TABLE stats_ext_tbl_range; +-- the order of columns and expressions in CREATE STATISTICS is preserved in the +-- object definition and the MCV; left undropped for pg_upgrade testing +CREATE TABLE declared_order_preserved (a int, c int) WITH (autovacuum_enabled = off); +INSERT INTO declared_order_preserved SELECT 1, 100 FROM generate_series(1, 100) g; +CREATE STATISTICS declared_order_preserved_stat ON c, (a + c), a FROM declared_order_preserved; +ANALYZE declared_order_preserved; +SELECT pg_get_statisticsobjdef(oid) FROM pg_statistic_ext + WHERE stxname = 'declared_order_preserved_stat'; + pg_get_statisticsobjdef +------------------------------------------------------------------------------------------------------- + CREATE STATISTICS public.declared_order_preserved_stat ON c, (a + c), a FROM declared_order_preserved +(1 row) + +-- MCV values in CREATE STATISTICS order: c = 100, (a + c) = 101, a = 1 +SELECT m.values +FROM pg_statistic_ext s +JOIN pg_statistic_ext_data d ON (s.oid = d.stxoid) +CROSS JOIN LATERAL pg_mcv_list_items(d.stxdmcv) m +WHERE s.stxname = 'declared_order_preserved_stat'; + values +------------- + {100,101,1} +(1 row) + +-- selectivity estimates are accurate regardless of the column order in CREATE +-- STATISTICS (here b, a) +CREATE TABLE declared_order_selectivity (a int, b int) WITH (autovacuum_enabled = off); +-- correlated data: (a, b) is either (0, 1) or (1, 0) +INSERT INTO declared_order_selectivity SELECT 0, 1 FROM generate_series(1, 9900); +INSERT INTO declared_order_selectivity SELECT 1, 0 FROM generate_series(1, 100); +CREATE STATISTICS declared_order_selectivity_stat ON b, a FROM declared_order_selectivity; +ANALYZE declared_order_selectivity; +-- estimate should track the actual 9900 rows +SELECT * FROM check_estimated_rows('SELECT * FROM declared_order_selectivity WHERE a = 0 AND b = 1'); + estimated | actual +-----------+-------- + 9900 | 9900 +(1 row) + +-- estimate should track the actual 100 rows +SELECT * FROM check_estimated_rows('SELECT * FROM declared_order_selectivity WHERE a = 1 AND b = 0'); + estimated | actual +-----------+-------- + 100 | 100 +(1 row) + +DROP TABLE declared_order_selectivity; +-- grouping estimates are accurate regardless of the column order in CREATE +-- STATISTICS (here b, a) +CREATE TABLE declared_order_grouping (a int, b int) WITH (autovacuum_enabled = off); +-- correlated data: a and b move together, so 100 distinct (a, b) groups +INSERT INTO declared_order_grouping SELECT i%100, i%100 FROM generate_series(1, 10000) i; +CREATE STATISTICS declared_order_grouping_stat ON b, a FROM declared_order_grouping; +ANALYZE declared_order_grouping; +-- estimate should track the 100 distinct groups, not ndistinct(a) * ndistinct(b) +SELECT * FROM check_estimated_rows('SELECT COUNT(*) FROM declared_order_grouping GROUP BY a, b'); + estimated | actual +-----------+-------- + 100 | 100 +(1 row) + +DROP TABLE declared_order_grouping; diff --git a/src/test/regress/expected/stats_import.out b/src/test/regress/expected/stats_import.out index 8dcec57cc30..d7c7fcc5793 100644 --- a/src/test/regress/expected/stats_import.out +++ b/src/test/regress/expected/stats_import.out @@ -78,7 +78,7 @@ SELECT COUNT(*) FROM pg_attribute attnum > 0; count ------- - 17 + 16 (1 row) -- Create a view that is used purely for the type based on pg_stats_ext. @@ -3585,6 +3585,119 @@ FROM stats_import.pg_stats_ext_exprs_get_difference('test_mr_stat', 'test_mr_sta \gx (0 rows) +-- import of extended statistics that include a virtual generated column +CREATE TABLE stats_import.test_virtual_gen( + a int, + b int, + c int GENERATED ALWAYS AS (a + b) VIRTUAL +); +INSERT INTO stats_import.test_virtual_gen(a, b) + SELECT mod(i, 10), mod(i, 7) FROM generate_series(1, 1000) s(i); +CREATE STATISTICS stats_import.test_virtual_gen_stat ON a, c + FROM stats_import.test_virtual_gen; +ANALYZE stats_import.test_virtual_gen; +CREATE TABLE stats_import.test_virtual_gen_clone( + a int, + b int, + c int GENERATED ALWAYS AS (a + b) VIRTUAL +); +CREATE STATISTICS stats_import.test_virtual_gen_stat_clone ON a, c + FROM stats_import.test_virtual_gen_clone; +-- Import stats from test_virtual_gen_stat to test_virtual_gen_stat_clone +SELECT e.statistics_name, + pg_catalog.pg_restore_extended_stats( + 'schemaname', e.statistics_schemaname::text, + 'relname', 'test_virtual_gen_clone', + 'statistics_schemaname', e.statistics_schemaname::text, + 'statistics_name', 'test_virtual_gen_stat_clone', + 'inherited', e.inherited, + 'n_distinct', e.n_distinct, + 'dependencies', e.dependencies, + 'most_common_vals', e.most_common_vals, + 'most_common_freqs', e.most_common_freqs, + 'most_common_base_freqs', e.most_common_base_freqs, + 'exprs', x.exprs) +FROM pg_stats_ext AS e +CROSS JOIN LATERAL ( + SELECT jsonb_agg(jsonb_strip_nulls(jsonb_build_object( + 'null_frac', ee.null_frac::text, + 'avg_width', ee.avg_width::text, + 'n_distinct', ee.n_distinct::text, + 'most_common_vals', ee.most_common_vals::text, + 'most_common_freqs', ee.most_common_freqs::text, + 'histogram_bounds', ee.histogram_bounds::text, + 'correlation', ee.correlation::text))) + FROM pg_stats_ext_exprs AS ee + WHERE ee.statistics_schemaname = e.statistics_schemaname AND + ee.statistics_name = e.statistics_name AND + ee.inherited = e.inherited + ) AS x(exprs) +WHERE e.statistics_schemaname = 'stats_import' +AND e.statistics_name = 'test_virtual_gen_stat'; + statistics_name | pg_restore_extended_stats +-----------------------+--------------------------- + test_virtual_gen_stat | t +(1 row) + +SELECT statname, (stats).* +FROM stats_import.pg_stats_ext_get_difference('test_virtual_gen_stat', 'test_virtual_gen_stat_clone') +\gx +(0 rows) + +SELECT statname, (stats).* +FROM stats_import.pg_stats_ext_exprs_get_difference('test_virtual_gen_stat', 'test_virtual_gen_stat_clone') +\gx +(0 rows) + +-- round-trip an MCV that interleaves a column and an expression of different types +CREATE STATISTICS stats_import.test_stat_interleave (mcv) + ON name, lower(arange), comp + FROM stats_import.test; +CREATE STATISTICS stats_import.test_stat_interleave_clone (mcv) + ON name, lower(arange), comp + FROM stats_import.test_clone; +ANALYZE stats_import.test; +-- Copy stats from test_stat_interleave to test_stat_interleave_clone +SELECT e.statistics_name, + pg_catalog.pg_restore_extended_stats( + 'schemaname', e.statistics_schemaname::text, + 'relname', 'test_clone', + 'statistics_schemaname', e.statistics_schemaname::text, + 'statistics_name', 'test_stat_interleave_clone', + 'inherited', e.inherited, + 'n_distinct', e.n_distinct, + 'dependencies', e.dependencies, + 'most_common_vals', e.most_common_vals, + 'most_common_freqs', e.most_common_freqs, + 'most_common_base_freqs', e.most_common_base_freqs, + 'exprs', x.exprs) +FROM pg_stats_ext AS e +CROSS JOIN LATERAL ( + SELECT jsonb_agg(jsonb_strip_nulls(jsonb_build_object( + 'null_frac', ee.null_frac::text, + 'avg_width', ee.avg_width::text, + 'n_distinct', ee.n_distinct::text, + 'most_common_vals', ee.most_common_vals::text, + 'most_common_freqs', ee.most_common_freqs::text, + 'histogram_bounds', ee.histogram_bounds::text, + 'correlation', ee.correlation::text))) + FROM pg_stats_ext_exprs AS ee + WHERE ee.statistics_schemaname = e.statistics_schemaname AND + ee.statistics_name = e.statistics_name AND + ee.inherited = e.inherited + ) AS x(exprs) +WHERE e.statistics_schemaname = 'stats_import' +AND e.statistics_name = 'test_stat_interleave'; + statistics_name | pg_restore_extended_stats +----------------------+--------------------------- + test_stat_interleave | t +(1 row) + +SELECT statname, (stats).* +FROM stats_import.pg_stats_ext_get_difference('test_stat_interleave', 'test_stat_interleave_clone') +\gx +(0 rows) + -- range_length_histogram, range_empty_frac, and range_bounds_histogram -- have been added to pg_stats_ext_exprs in PostgreSQL 19. When dumping -- expression statistics in a cluster with an older version, these fields @@ -3671,7 +3784,7 @@ SELECT COUNT(*) FROM stats_import.test_range_expr_null (1 row) DROP SCHEMA stats_import CASCADE; -NOTICE: drop cascades to 19 other objects +NOTICE: drop cascades to 21 other objects DETAIL: drop cascades to view stats_import.pg_stats_stable drop cascades to view stats_import.pg_statistic_flat_t drop cascades to function stats_import.pg_statistic_flat(text) @@ -3690,4 +3803,6 @@ drop cascades to sequence stats_import.testseq drop cascades to view stats_import.testview drop cascades to table stats_import.test_clone drop cascades to table stats_import.test_mr_clone +drop cascades to table stats_import.test_virtual_gen +drop cascades to table stats_import.test_virtual_gen_clone drop cascades to table stats_import.test_range_expr_null diff --git a/src/test/regress/sql/create_table_like.sql b/src/test/regress/sql/create_table_like.sql index d52a93ef131..864f54d0e6f 100644 --- a/src/test/regress/sql/create_table_like.sql +++ b/src/test/regress/sql/create_table_like.sql @@ -277,7 +277,7 @@ SELECT attname, attcompression FROM pg_attribute WHERE attrelid = 'ctl_foreign_table2'::regclass and attnum > 0 ORDER BY attnum; -- LIKE ... INCLUDING STATISTICS with dropped columns in the parent, --- so stxkeys attnums are not contiguous. +-- so column attnums are not contiguous. CREATE TABLE ctl_stats3_parent (a int, b int, c int); ALTER TABLE ctl_stats3_parent DROP COLUMN b; CREATE STATISTICS ctl_stats3_stat ON a, c FROM ctl_stats3_parent; @@ -287,15 +287,10 @@ ALTER TABLE ctl_stats4_parent DROP COLUMN b; CREATE STATISTICS ctl_stats4_stat ON a, c FROM ctl_stats4_parent; CREATE TABLE ctl_stats4_child (LIKE ctl_stats4_parent INCLUDING STATISTICS); SELECT s.stxrelid::regclass AS relation, - array_agg(a.attname ORDER BY u.ord) AS stats_columns + pg_get_statisticsobjdef_columns(s.oid) AS stats_columns FROM pg_statistic_ext s -CROSS JOIN LATERAL - unnest(s.stxkeys::int2[]) WITH ORDINALITY AS u(attnum, ord) -JOIN pg_attribute a - ON a.attrelid = s.stxrelid AND a.attnum = u.attnum WHERE s.stxrelid IN ('ctl_stats3_child'::regclass, 'ctl_stats4_child'::regclass) -GROUP BY s.stxrelid ORDER BY s.stxrelid::regclass::text; DROP TABLE ctl_stats3_parent; DROP TABLE ctl_stats3_child; diff --git a/src/test/regress/sql/stats_ext.sql b/src/test/regress/sql/stats_ext.sql index 3cc6012b822..33844520303 100644 --- a/src/test/regress/sql/stats_ext.sql +++ b/src/test/regress/sql/stats_ext.sql @@ -178,6 +178,35 @@ SELECT * FROM check_estimated_rows('SELECT a, b FROM stxdinp GROUP BY 1, 2'); SELECT * FROM check_estimated_rows('SELECT a + 1, b FROM ONLY stxdinp GROUP BY 1, 2'); DROP TABLE stxdinp; +-- a column-equivalent expression (one that const-folds to a plain column, or a +-- passthrough virtual generated column) is treated as that column +CREATE TABLE column_equivalent (a int, b int, g int GENERATED ALWAYS AS (a) VIRTUAL) WITH (autovacuum_enabled = off); +INSERT INTO column_equivalent (a, b) SELECT i%100, i%10 FROM generate_series(1, 10000) i; +-- a column-equivalent CASE between two complex expressions +CREATE STATISTICS column_equivalent_middle ON (a + b), (CASE WHEN true THEN a ELSE b END), (a * b) FROM column_equivalent; +-- only column-equivalent entries, which all fold to plain columns +CREATE STATISTICS column_equivalent_all ON (CASE WHEN true THEN a END), b FROM column_equivalent; +ANALYZE column_equivalent; +-- a column-equivalent CASE is not reported as an expression; only the complex +-- expressions appear, each with its own statistics +SELECT statistics_name, expr, n_distinct +FROM pg_stats_ext_exprs +WHERE statistics_name IN ('column_equivalent_middle', 'column_equivalent_all') +ORDER BY statistics_name, expr; +-- each complex expression's GROUP BY estimate uses its own statistics +SELECT * FROM check_estimated_rows('SELECT (a + b) FROM column_equivalent GROUP BY 1'); +SELECT * FROM check_estimated_rows('SELECT (a * b) FROM column_equivalent GROUP BY 1'); +-- column_equivalent_all is used for a GROUP BY on the plain columns a, b +SELECT * FROM check_estimated_rows('SELECT COUNT(*) FROM column_equivalent GROUP BY a, b'); +-- an entry that reduces to a column already listed is rejected as a duplicate: +-- a parenthesized column, +CREATE STATISTICS column_equivalent_dup_paren ON a, (a) FROM column_equivalent; +-- a constant-folding expression, +CREATE STATISTICS column_equivalent_dup_case ON a, (CASE WHEN true THEN a END), b FROM column_equivalent; +-- and a passthrough generated column +CREATE STATISTICS column_equivalent_dup_vgen ON a, g FROM column_equivalent; +DROP TABLE column_equivalent; + -- basic test for statistics on expressions CREATE TABLE ab1 (a INTEGER, b INTEGER, c TIMESTAMP, d TIMESTAMPTZ); @@ -1593,6 +1622,12 @@ ANALYZE virtual_gen_stats; SELECT * FROM check_estimated_rows('SELECT * FROM virtual_gen_stats WHERE c = 0 AND (3*b) = 0'); SELECT * FROM check_estimated_rows('SELECT * FROM virtual_gen_stats WHERE d = 0 AND (d-2*a) = 0'); +SELECT expr FROM pg_stats_ext_exprs + WHERE statistics_name = 'virtual_gen_stats_1' AND NOT inherited; + +SELECT pg_get_statisticsobjdef(oid) FROM pg_statistic_ext + WHERE stxname = 'virtual_gen_stats_1'; + -- univariate statistics on individual virtual generated columns DROP STATISTICS virtual_gen_stats_1; @@ -1901,10 +1936,50 @@ CREATE STATISTICS stats_ext_range (mcv) ON irange, (irange + '[4,10)'::int4range) FROM stats_ext_tbl_range; ANALYZE stats_ext_tbl_range; -SELECT attnames, most_common_vals +SELECT exprs, most_common_vals FROM pg_stats_ext WHERE statistics_name = 'stats_ext_range'; SELECT range_length_histogram, range_empty_frac, range_bounds_histogram FROM pg_stats_ext_exprs WHERE statistics_name = 'stats_ext_range'; DROP TABLE stats_ext_tbl_range; + +-- the order of columns and expressions in CREATE STATISTICS is preserved in the +-- object definition and the MCV; left undropped for pg_upgrade testing +CREATE TABLE declared_order_preserved (a int, c int) WITH (autovacuum_enabled = off); +INSERT INTO declared_order_preserved SELECT 1, 100 FROM generate_series(1, 100) g; +CREATE STATISTICS declared_order_preserved_stat ON c, (a + c), a FROM declared_order_preserved; +ANALYZE declared_order_preserved; +SELECT pg_get_statisticsobjdef(oid) FROM pg_statistic_ext + WHERE stxname = 'declared_order_preserved_stat'; +-- MCV values in CREATE STATISTICS order: c = 100, (a + c) = 101, a = 1 +SELECT m.values +FROM pg_statistic_ext s +JOIN pg_statistic_ext_data d ON (s.oid = d.stxoid) +CROSS JOIN LATERAL pg_mcv_list_items(d.stxdmcv) m +WHERE s.stxname = 'declared_order_preserved_stat'; + +-- selectivity estimates are accurate regardless of the column order in CREATE +-- STATISTICS (here b, a) +CREATE TABLE declared_order_selectivity (a int, b int) WITH (autovacuum_enabled = off); +-- correlated data: (a, b) is either (0, 1) or (1, 0) +INSERT INTO declared_order_selectivity SELECT 0, 1 FROM generate_series(1, 9900); +INSERT INTO declared_order_selectivity SELECT 1, 0 FROM generate_series(1, 100); +CREATE STATISTICS declared_order_selectivity_stat ON b, a FROM declared_order_selectivity; +ANALYZE declared_order_selectivity; +-- estimate should track the actual 9900 rows +SELECT * FROM check_estimated_rows('SELECT * FROM declared_order_selectivity WHERE a = 0 AND b = 1'); +-- estimate should track the actual 100 rows +SELECT * FROM check_estimated_rows('SELECT * FROM declared_order_selectivity WHERE a = 1 AND b = 0'); +DROP TABLE declared_order_selectivity; + +-- grouping estimates are accurate regardless of the column order in CREATE +-- STATISTICS (here b, a) +CREATE TABLE declared_order_grouping (a int, b int) WITH (autovacuum_enabled = off); +-- correlated data: a and b move together, so 100 distinct (a, b) groups +INSERT INTO declared_order_grouping SELECT i%100, i%100 FROM generate_series(1, 10000) i; +CREATE STATISTICS declared_order_grouping_stat ON b, a FROM declared_order_grouping; +ANALYZE declared_order_grouping; +-- estimate should track the 100 distinct groups, not ndistinct(a) * ndistinct(b) +SELECT * FROM check_estimated_rows('SELECT COUNT(*) FROM declared_order_grouping GROUP BY a, b'); +DROP TABLE declared_order_grouping; diff --git a/src/test/regress/sql/stats_import.sql b/src/test/regress/sql/stats_import.sql index 10843f12447..c165738f05c 100644 --- a/src/test/regress/sql/stats_import.sql +++ b/src/test/regress/sql/stats_import.sql @@ -2477,6 +2477,108 @@ SELECT statname, (stats).* FROM stats_import.pg_stats_ext_exprs_get_difference('test_mr_stat', 'test_mr_stat_clone') \gx +-- import of extended statistics that include a virtual generated column +CREATE TABLE stats_import.test_virtual_gen( + a int, + b int, + c int GENERATED ALWAYS AS (a + b) VIRTUAL +); +INSERT INTO stats_import.test_virtual_gen(a, b) + SELECT mod(i, 10), mod(i, 7) FROM generate_series(1, 1000) s(i); +CREATE STATISTICS stats_import.test_virtual_gen_stat ON a, c + FROM stats_import.test_virtual_gen; +ANALYZE stats_import.test_virtual_gen; +CREATE TABLE stats_import.test_virtual_gen_clone( + a int, + b int, + c int GENERATED ALWAYS AS (a + b) VIRTUAL +); +CREATE STATISTICS stats_import.test_virtual_gen_stat_clone ON a, c + FROM stats_import.test_virtual_gen_clone; +-- Import stats from test_virtual_gen_stat to test_virtual_gen_stat_clone +SELECT e.statistics_name, + pg_catalog.pg_restore_extended_stats( + 'schemaname', e.statistics_schemaname::text, + 'relname', 'test_virtual_gen_clone', + 'statistics_schemaname', e.statistics_schemaname::text, + 'statistics_name', 'test_virtual_gen_stat_clone', + 'inherited', e.inherited, + 'n_distinct', e.n_distinct, + 'dependencies', e.dependencies, + 'most_common_vals', e.most_common_vals, + 'most_common_freqs', e.most_common_freqs, + 'most_common_base_freqs', e.most_common_base_freqs, + 'exprs', x.exprs) +FROM pg_stats_ext AS e +CROSS JOIN LATERAL ( + SELECT jsonb_agg(jsonb_strip_nulls(jsonb_build_object( + 'null_frac', ee.null_frac::text, + 'avg_width', ee.avg_width::text, + 'n_distinct', ee.n_distinct::text, + 'most_common_vals', ee.most_common_vals::text, + 'most_common_freqs', ee.most_common_freqs::text, + 'histogram_bounds', ee.histogram_bounds::text, + 'correlation', ee.correlation::text))) + FROM pg_stats_ext_exprs AS ee + WHERE ee.statistics_schemaname = e.statistics_schemaname AND + ee.statistics_name = e.statistics_name AND + ee.inherited = e.inherited + ) AS x(exprs) +WHERE e.statistics_schemaname = 'stats_import' +AND e.statistics_name = 'test_virtual_gen_stat'; + +SELECT statname, (stats).* +FROM stats_import.pg_stats_ext_get_difference('test_virtual_gen_stat', 'test_virtual_gen_stat_clone') +\gx + +SELECT statname, (stats).* +FROM stats_import.pg_stats_ext_exprs_get_difference('test_virtual_gen_stat', 'test_virtual_gen_stat_clone') +\gx + +-- round-trip an MCV that interleaves a column and an expression of different types +CREATE STATISTICS stats_import.test_stat_interleave (mcv) + ON name, lower(arange), comp + FROM stats_import.test; +CREATE STATISTICS stats_import.test_stat_interleave_clone (mcv) + ON name, lower(arange), comp + FROM stats_import.test_clone; +ANALYZE stats_import.test; +-- Copy stats from test_stat_interleave to test_stat_interleave_clone +SELECT e.statistics_name, + pg_catalog.pg_restore_extended_stats( + 'schemaname', e.statistics_schemaname::text, + 'relname', 'test_clone', + 'statistics_schemaname', e.statistics_schemaname::text, + 'statistics_name', 'test_stat_interleave_clone', + 'inherited', e.inherited, + 'n_distinct', e.n_distinct, + 'dependencies', e.dependencies, + 'most_common_vals', e.most_common_vals, + 'most_common_freqs', e.most_common_freqs, + 'most_common_base_freqs', e.most_common_base_freqs, + 'exprs', x.exprs) +FROM pg_stats_ext AS e +CROSS JOIN LATERAL ( + SELECT jsonb_agg(jsonb_strip_nulls(jsonb_build_object( + 'null_frac', ee.null_frac::text, + 'avg_width', ee.avg_width::text, + 'n_distinct', ee.n_distinct::text, + 'most_common_vals', ee.most_common_vals::text, + 'most_common_freqs', ee.most_common_freqs::text, + 'histogram_bounds', ee.histogram_bounds::text, + 'correlation', ee.correlation::text))) + FROM pg_stats_ext_exprs AS ee + WHERE ee.statistics_schemaname = e.statistics_schemaname AND + ee.statistics_name = e.statistics_name AND + ee.inherited = e.inherited + ) AS x(exprs) +WHERE e.statistics_schemaname = 'stats_import' +AND e.statistics_name = 'test_stat_interleave'; + +SELECT statname, (stats).* +FROM stats_import.pg_stats_ext_get_difference('test_stat_interleave', 'test_stat_interleave_clone') +\gx + -- range_length_histogram, range_empty_frac, and range_bounds_histogram -- have been added to pg_stats_ext_exprs in PostgreSQL 19. When dumping -- expression statistics in a cluster with an older version, these fields -- 2.50.1 (Apple Git-155)