From 952eb9ac92a13f5f6c09d1411659d5c18852bea8 Mon Sep 17 00:00:00 2001 From: Alexandra Wang Date: Mon, 3 Aug 2026 15:59:52 -0700 Subject: [PATCH v10 2/4] Unify extended statistics columns and expressions into a single stxexprs list Remove the stxkeys int2vector from pg_statistic_ext. A statistics object's columns and expressions are now stored together in stxexprs, in user-written order, with plain columns represented as Var nodes. The columns previously lived in a separate stxkeys array, sorted by attnum, which lost that order. This leaves one representation of them instead of two, in the catalog and in the in-memory StatExtEntry and StatisticExtInfo structs alike. The pg_stats_ext view accordingly drops its attnames column and shows all entries in exprs. Where the code previously kept a keys Bitmapset beside the expression list, StatExtEntry and StatisticExtInfo now carry a Bitmapset of the positions in the list that hold plain columns; the covered column attnums and the per-expression index are derived from the list and that set. The user-written order is preserved in the pg_stats_ext view, in the CREATE STATISTICS command that pg_dump reproduces, and in the MCV list. MCV dimensions are positional, so the planner maps a clause to its dimension through the list instead of assuming columns come first. The ndistinct and dependencies data identify their attributes by number, so their order within an item does not affect estimation, but pg_dump and pg_upgrade reload that data through the pg_ndistinct and pg_dependencies input functions, which require a canonical order (columns ascending, then expressions descending). Attnum-sorted stxkeys used to produce that order; statext_ndistinct_build and statext_dependencies_build now sort each item explicitly. This is a prerequisite refactor for the join statistics patch. Suggested-by: Tom Lane Suggested-by: Tomas Vondra Discussion: https://postgr.es/m/711247.1779913876@sss.pgh.pa.us --- doc/src/sgml/catalogs.sgml | 19 +- doc/src/sgml/perform.sgml | 116 ++-- doc/src/sgml/system-views.sgml | 13 +- src/backend/catalog/system_views.sql | 7 +- src/backend/commands/statscmds.c | 200 +++---- src/backend/optimizer/util/plancat.c | 100 ++-- src/backend/parser/parse_utilcmd.c | 65 +-- src/backend/statistics/dependencies.c | 61 +- src/backend/statistics/extended_stats.c | 551 +++++++++++------- src/backend/statistics/extended_stats_funcs.c | 160 +++-- src/backend/statistics/mcv.c | 75 +-- src/backend/statistics/mvdistinct.c | 23 +- src/backend/utils/adt/ruleutils.c | 297 +++++----- src/backend/utils/adt/selfuncs.c | 23 +- src/bin/psql/describe.c | 18 +- src/include/catalog/pg_proc.dat | 4 +- src/include/catalog/pg_statistic_ext.h | 15 +- src/include/nodes/pathnodes.h | 8 +- .../statistics/extended_stats_internal.h | 7 +- src/include/statistics/statistics.h | 6 + .../regress/expected/create_table_like.out | 9 +- src/test/regress/expected/oidjoins.out | 1 - src/test/regress/expected/rules.out | 5 +- src/test/regress/expected/stats_ext.out | 193 +++++- src/test/regress/expected/stats_import.out | 161 ++++- src/test/regress/sql/create_table_like.sql | 9 +- src/test/regress/sql/stats_ext.sql | 99 +++- src/test/regress/sql/stats_import.sql | 123 ++++ 28 files changed, 1486 insertions(+), 882 deletions(-) diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index 07ba05dccd9..3703858b2a7 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -7836,18 +7836,6 @@ SCRAM-SHA-256$<iteration count>:&l - - - stxkeys int2vector - (references pg_attribute.attnum) - - - An array of attribute numbers, indicating which table columns are - covered by this statistics object; - for example a value of 1 3 would - mean that the first and the third table columns are covered - - @@ -7886,9 +7874,10 @@ SCRAM-SHA-256$<iteration count>:&l Expression trees (in nodeToString() - representation) for statistics object attributes that are not simple - column references. This is a list with one element per expression. - Null if all statistics object attributes are simple references. + representation) for the columns and expressions the statistics object + is defined on. Both column + references and expressions are stored here, with one element per + column or expression, in the order they were written. diff --git a/doc/src/sgml/perform.sgml b/doc/src/sgml/perform.sgml index ea8da01b779..b4e695a263d 100644 --- a/doc/src/sgml/perform.sgml +++ b/doc/src/sgml/perform.sgml @@ -1542,30 +1542,32 @@ CREATE STATISTICS stts (dependencies) ON city, zip FROM zipcodes; ANALYZE zipcodes; -SELECT stxkeys AS k, jsonb_pretty(stxddependencies::text::jsonb) AS dep +SELECT jsonb_pretty(stxddependencies::text::jsonb) AS dep FROM pg_statistic_ext JOIN pg_statistic_ext_data ON (oid = stxoid) WHERE stxname = 'stts'; --[ RECORD 1 ]-------------------- -k | 1 5 -dep | [ + - | { + - | "degree": 1.000000,+ - | "attributes": [ + - | 1 + - | ], + - | "dependency": 5 + - | }, + - | { + - | "degree": 0.423130,+ - | "attributes": [ + - | 5 + - | ], + - | "dependency": 1 + - | } + - | ] + dep +----------------------------- + [ + + { + + "degree": 1.000000,+ + "attributes": [ + + 1 + + ], + + "dependency": 5 + + }, + + { + + "degree": 0.423130,+ + "attributes": [ + + 5 + + ], + + "dependency": 1 + + } + + ] (1 row) - Here it can be seen that column 1 (zip code) fully determines column + The dependency data identifies columns by attribute number: in this + table zip is attribute 1 and city is attribute 5. It can then be seen + that column 1 (zip code) fully determines column 5 (city) so the coefficient is 1.0, while city only determines zip code about 42% of the time, meaning that there are many cities (58%) that are represented by more than a single ZIP code. @@ -1648,46 +1650,48 @@ CREATE STATISTICS stts2 (ndistinct) ON city, state, zip FROM zipcodes; ANALYZE zipcodes; -SELECT stxkeys AS k, jsonb_pretty(stxdndistinct::text::jsonb) AS nd +SELECT jsonb_pretty(stxdndistinct::text::jsonb) AS nd FROM pg_statistic_ext JOIN pg_statistic_ext_data on (oid = stxoid) WHERE stxname = 'stts2'; --[ RECORD 1 ]------------------- -k | 1 2 5 -nd | [ + - | { + - | "ndistinct": 33178,+ - | "attributes": [ + - | 1, + - | 2 + - | ] + - | }, + - | { + - | "ndistinct": 33178,+ - | "attributes": [ + - | 1, + - | 5 + - | ] + - | }, + - | { + - | "ndistinct": 27435,+ - | "attributes": [ + - | 2, + - | 5 + - | ] + - | }, + - | { + - | "ndistinct": 33178,+ - | "attributes": [ + - | 1, + - | 2, + - | 5 + - | ] + - | } + - | ] + nd +----------------------------- + [ + + { + + "ndistinct": 33178,+ + "attributes": [ + + 1, + + 2 + + ] + + }, + + { + + "ndistinct": 33178,+ + "attributes": [ + + 1, + + 5 + + ] + + }, + + { + + "ndistinct": 27435,+ + "attributes": [ + + 2, + + 5 + + ] + + }, + + { + + "ndistinct": 33178,+ + "attributes": [ + + 1, + + 2, + + 5 + + ] + + } + + ] (1 row) - This indicates that there are three combinations of columns that - have 33178 distinct values: ZIP code and state; ZIP code and city; + As above, the columns appear by attribute number: zip is 1, state is 2 + and city is 5. This indicates that there are three combinations of + columns that have 33178 distinct values: ZIP code and state; ZIP code + and city; and ZIP code, city and state (the fact that they are all equal is expected given that ZIP code alone is unique in this table). On the other hand, the combination of city and state has only 27435 distinct diff --git a/doc/src/sgml/system-views.sgml b/doc/src/sgml/system-views.sgml index 77202e2c765..4d9b6221b07 100644 --- a/doc/src/sgml/system-views.sgml +++ b/doc/src/sgml/system-views.sgml @@ -4737,22 +4737,13 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx - - - attnames name[] - (references pg_attribute.attname) - - - Names of the columns included in the extended statistics object - - - exprs text[] - Expressions included in the extended statistics object + The columns and expressions the statistics object is + defined on, in the order they were written diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 6dce9192ba0..f1e16db7340 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -285,12 +285,7 @@ CREATE VIEW pg_stats_ext WITH (security_barrier) AS s.stxname AS statistics_name, s.oid AS statistics_id, pg_get_userbyid(s.stxowner) AS statistics_owner, - ( SELECT array_agg(a.attname ORDER BY a.attnum) - FROM unnest(s.stxkeys) k - JOIN pg_attribute a - ON (a.attrelid = s.stxrelid AND a.attnum = k) - ) AS attnames, - pg_get_statisticsobjdef_expressions(s.oid) as exprs, + pg_get_statisticsobjdef_columns(s.oid) AS exprs, s.stxkind AS kinds, sd.stxdinherit AS inherited, sd.stxdndistinct AS n_distinct, diff --git a/src/backend/commands/statscmds.c b/src/backend/commands/statscmds.c index 4aaf41855ce..03dbf68f2a0 100644 --- a/src/backend/commands/statscmds.c +++ b/src/backend/commands/statscmds.c @@ -46,18 +46,6 @@ static char *ChooseExtendedStatisticName(const char *name1, const char *name2, const char *label, Oid namespaceid); static char *ChooseExtendedStatisticNameAddition(List *exprs); - -/* qsort comparator for the attnums in CreateStatistics */ -static int -compare_int16(const void *a, const void *b) -{ - int av = *(const int16 *) a; - int bv = *(const int16 *) b; - - /* this can't overflow if int is wider than int16 */ - return (av - bv); -} - /* * CREATE STATISTICS * @@ -70,8 +58,6 @@ compare_int16(const void *a, const void *b) ObjectAddress CreateStatistics(List *relids, CreateStatsStmt *stmt, bool check_rights) { - int16 attnums[STATS_MAX_DIMENSIONS]; - int nattnums = 0; int numcols; char *namestr; NameData stxname; @@ -81,8 +67,9 @@ CreateStatistics(List *relids, CreateStatsStmt *stmt, bool check_rights) HeapTuple htup; Datum values[Natts_pg_statistic_ext]; bool nulls[Natts_pg_statistic_ext]; - int2vector *stxkeys; List *stxexprs = NIL; + Bitmapset *columns; + char *exprsString; Datum exprsDatum; Relation statrel; Relation rel = NULL; @@ -96,8 +83,8 @@ CreateStatistics(List *relids, CreateStatsStmt *stmt, bool check_rights) bool build_dependencies; bool build_mcv; bool build_expressions; + bool has_simple_var = false; bool requested_type = false; - int i; ListCell *cell; ListCell *cell2; @@ -241,8 +228,8 @@ CreateStatistics(List *relids, CreateStatsStmt *stmt, bool check_rights) STATS_MAX_DIMENSIONS))); /* - * Convert the expression list to a simple array of attnums, but also keep - * a list of more complex expressions. While at it, enforce some + * Convert the expression list to a list of expression trees. Simple + * column references are stored as Var nodes. While at it, enforce some * constraints - we don't allow extended statistics on system attributes, * and we require the data type to have a less-than operator, if we're * building multivariate statistics. @@ -297,24 +284,14 @@ CreateStatistics(List *relids, CreateStatsStmt *stmt, bool check_rights) format_type_be(attForm->atttypid)))); } - /* Treat virtual generated columns as expressions */ - if (attForm->attgenerated == ATTRIBUTE_GENERATED_VIRTUAL) - { - Node *expr; - - expr = (Node *) makeVar(1, - attForm->attnum, - attForm->atttypid, - attForm->atttypmod, - attForm->attcollation, - 0); - stxexprs = lappend(stxexprs, expr); - } - else - { - attnums[nattnums] = attForm->attnum; - nattnums++; - } + stxexprs = lappend(stxexprs, + (Node *) makeVar(1, + attForm->attnum, + attForm->atttypid, + attForm->atttypmod, + attForm->attcollation, + 0)); + has_simple_var = true; ReleaseSysCache(atttuple); } else if (IsA(selem->expr, Var)) /* column reference in parens */ @@ -344,16 +321,8 @@ CreateStatistics(List *relids, CreateStatsStmt *stmt, bool check_rights) format_type_be(var->vartype)))); } - /* Treat virtual generated columns as expressions */ - if (get_attgenerated(relid, var->varattno) == ATTRIBUTE_GENERATED_VIRTUAL) - { - stxexprs = lappend(stxexprs, (Node *) var); - } - else - { - attnums[nattnums] = var->varattno; - nattnums++; - } + stxexprs = lappend(stxexprs, (Node *) var); + has_simple_var = true; } else /* expression */ { @@ -412,20 +381,23 @@ CreateStatistics(List *relids, CreateStatsStmt *stmt, bool check_rights) * that we're building statistics on a single expression (or virtual * generated column). */ - if (numcols < 2 && list_length(stxexprs) != 1) - ereport(ERROR, - errcode(ERRCODE_INVALID_OBJECT_DEFINITION), - errmsg("cannot create extended statistics on a single non-virtual column"), - errdetail("Univariate statistics are already built for each individual non-virtual table column.")); + if (numcols == 1) + { + Node *single = (Node *) linitial(stxexprs); - /* - * Parse the statistics kinds (not allowed when building univariate - * statistics). - */ - if (numcols == 1 && stmt->stat_types != NIL) - ereport(ERROR, - errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot specify statistics kinds when building univariate statistics")); + if (IsA(single, Var) && + get_attgenerated(relid, ((Var *) single)->varattno) != ATTRIBUTE_GENERATED_VIRTUAL) + ereport(ERROR, + errcode(ERRCODE_INVALID_OBJECT_DEFINITION), + errmsg("cannot create extended statistics on a single non-virtual column"), + errdetail("Univariate statistics are already built for each individual non-virtual table column.")); + + /* statistics kinds are not allowed with univariate statistics */ + if (stmt->stat_types != NIL) + ereport(ERROR, + errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot specify statistics kinds when building univariate statistics")); + } build_ndistinct = false; build_dependencies = false; @@ -468,44 +440,42 @@ CreateStatistics(List *relids, CreateStatsStmt *stmt, bool check_rights) } /* - * When there are non-trivial expressions, build the expression stats - * automatically. This allows calculating good estimates for stats that - * consider per-clause estimates (e.g. functional dependencies). - */ - build_expressions = (stxexprs != NIL); - - /* - * Sort the attnums, which makes detecting duplicates somewhat easier, and - * it does not hurt (it does not matter for the contents, unlike for - * indexes, for example). - */ - qsort(attnums, nattnums, sizeof(int16), compare_int16); - - /* - * Check for duplicates in the list of columns. The attnums are sorted so - * just check consecutive elements. + * Build per-expression statistics automatically when the object covers + * anything that has no pg_statistic entry of its own: complex + * expressions, or virtual generated columns (stored as Var nodes, but + * like expressions they get no pg_statistic row, so they need + * per-expression stats). Plain columns are skipped, since they already + * have univariate statistics in pg_statistic. These give good estimates + * where per-clause estimates matter (e.g. functional dependencies). */ - for (i = 1; i < nattnums; i++) + build_expressions = false; + foreach(cell, stxexprs) { - if (attnums[i] == attnums[i - 1]) - ereport(ERROR, - (errcode(ERRCODE_DUPLICATE_COLUMN), - errmsg("duplicate column name in statistics definition"))); + Node *expr = (Node *) lfirst(cell); + + if (!IsA(expr, Var) || + get_attgenerated(relid, ((Var *) expr)->varattno) == ATTRIBUTE_GENERATED_VIRTUAL) + { + build_expressions = true; + break; + } } /* - * Check for duplicate expressions. We do two loops, counting the - * occurrences of each expression. This is O(N^2) but we only allow small - * number of expressions and it's not executed often. + * Check for duplicate entries. We do two loops, counting the occurrences + * of each entry. This is O(N^2) but we only allow small number of entries + * and it's not executed often. * * XXX We don't cross-check attributes and expressions, because it does - * not seem worth it. In principle we could check that expressions don't - * contain trivial attribute references like "(a)", but the reasoning is - * similar to why we don't bother with extracting columns from - * expressions. It's either expensive or very easy to defeat for + * not seem worth it. In principle we could check that expressions aren't + * equivalent to trivial attribute references like "(a+0)", but the + * reasoning is similar to why we don't bother with extracting columns + * from expressions. It's either expensive or very easy to defeat for * determined user, and there's no risk if we allow such statistics (the * statistics is useless, but harmless). */ + columns = statext_column_entries(stxexprs, rel); + foreach(cell, stxexprs) { Node *expr1 = (Node *) lfirst(cell); @@ -519,18 +489,19 @@ CreateStatistics(List *relids, CreateStatsStmt *stmt, bool check_rights) cnt += 1; } - /* every expression should find at least itself */ + /* every entry should find at least itself */ Assert(cnt >= 1); + if (cnt > 1 && bms_is_member(foreach_current_index(cell), columns)) + ereport(ERROR, + (errcode(ERRCODE_DUPLICATE_COLUMN), + errmsg("duplicate column name in statistics definition"))); if (cnt > 1) ereport(ERROR, (errcode(ERRCODE_DUPLICATE_COLUMN), errmsg("duplicate expression in statistics definition"))); } - /* Form an int2vector representation of the sorted column list */ - stxkeys = buildint2vector(attnums, nattnums); - /* construct the char array of enabled statistic types */ ntypes = 0; if (build_ndistinct) @@ -544,17 +515,10 @@ CreateStatistics(List *relids, CreateStatsStmt *stmt, bool check_rights) Assert(ntypes > 0 && ntypes <= lengthof(types)); stxkind = construct_array_builtin(types, ntypes, CHAROID); - /* convert the expressions (if any) to a text datum */ - if (stxexprs != NIL) - { - char *exprsString; - - exprsString = nodeToString(stxexprs); - exprsDatum = CStringGetTextDatum(exprsString); - pfree(exprsString); - } - else - exprsDatum = (Datum) 0; + /* convert the expression list to a text datum */ + exprsString = nodeToString(stxexprs); + exprsDatum = CStringGetTextDatum(exprsString); + pfree(exprsString); statrel = table_open(StatisticExtRelationId, RowExclusiveLock); @@ -571,13 +535,9 @@ CreateStatistics(List *relids, CreateStatsStmt *stmt, bool check_rights) values[Anum_pg_statistic_ext_stxname - 1] = NameGetDatum(&stxname); values[Anum_pg_statistic_ext_stxnamespace - 1] = ObjectIdGetDatum(namespaceId); values[Anum_pg_statistic_ext_stxowner - 1] = ObjectIdGetDatum(stxowner); - values[Anum_pg_statistic_ext_stxkeys - 1] = PointerGetDatum(stxkeys); nulls[Anum_pg_statistic_ext_stxstattarget - 1] = true; values[Anum_pg_statistic_ext_stxkind - 1] = PointerGetDatum(stxkind); - values[Anum_pg_statistic_ext_stxexprs - 1] = exprsDatum; - if (exprsDatum == (Datum) 0) - nulls[Anum_pg_statistic_ext_stxexprs - 1] = true; /* insert it into pg_statistic_ext */ htup = heap_form_tuple(statrel->rd_att, values, nulls); @@ -607,13 +567,6 @@ CreateStatistics(List *relids, CreateStatsStmt *stmt, bool check_rights) */ ObjectAddressSet(myself, StatisticExtRelationId, statoid); - /* add dependencies for plain column references */ - for (i = 0; i < nattnums; i++) - { - ObjectAddressSubSet(parentobject, RelationRelationId, relid, attnums[i]); - recordDependencyOn(&myself, &parentobject, DEPENDENCY_AUTO); - } - /* * If there are no dependencies on a column, give the statistics object an * auto dependency on the whole table. In most cases, this will be @@ -625,7 +578,7 @@ CreateStatistics(List *relids, CreateStatsStmt *stmt, bool check_rights) * dependency, because recordDependencyOnSingleRelExpr may not create any * dependencies for whole-row Vars. */ - if (!nattnums) + if (!has_simple_var) { ObjectAddressSet(parentobject, RelationRelationId, relid); recordDependencyOn(&myself, &parentobject, DEPENDENCY_AUTO); @@ -635,17 +588,14 @@ CreateStatistics(List *relids, CreateStatsStmt *stmt, bool check_rights) * Store dependencies on anything mentioned in statistics expressions, * just like we do for index expressions. */ - if (stxexprs) - { - if (check_rights) - CheckUsageOnTypesInSingleRelExpr((Node *) stxexprs, relid, GetUserId()); - - recordDependencyOnSingleRelExpr(&myself, - (Node *) stxexprs, - relid, - DEPENDENCY_NORMAL, - DEPENDENCY_AUTO, false); - } + if (check_rights) + CheckUsageOnTypesInSingleRelExpr((Node *) stxexprs, relid, GetUserId()); + + recordDependencyOnSingleRelExpr(&myself, + (Node *) stxexprs, + relid, + DEPENDENCY_NORMAL, + DEPENDENCY_AUTO, false); /* * Also add dependencies on namespace and owner. These are required diff --git a/src/backend/optimizer/util/plancat.c b/src/backend/optimizer/util/plancat.c index 22e0e8b11f8..f7d4454f232 100644 --- a/src/backend/optimizer/util/plancat.c +++ b/src/backend/optimizer/util/plancat.c @@ -1655,7 +1655,7 @@ get_relation_constraints(PlannerInfo *root, static void get_relation_statistics_worker(List **stainfos, RelOptInfo *rel, Oid statOid, bool inh, - Bitmapset *keys, List *exprs) + List *exprs, Bitmapset *columns) { Form_pg_statistic_ext_data dataForm; HeapTuple dtup; @@ -1676,8 +1676,8 @@ get_relation_statistics_worker(List **stainfos, RelOptInfo *rel, info->inherit = dataForm->stxdinherit; info->rel = rel; info->kind = STATS_EXT_NDISTINCT; - info->keys = bms_copy(keys); info->exprs = exprs; + info->columns = columns; *stainfos = lappend(*stainfos, info); } @@ -1690,8 +1690,8 @@ get_relation_statistics_worker(List **stainfos, RelOptInfo *rel, info->inherit = dataForm->stxdinherit; info->rel = rel; info->kind = STATS_EXT_DEPENDENCIES; - info->keys = bms_copy(keys); info->exprs = exprs; + info->columns = columns; *stainfos = lappend(*stainfos, info); } @@ -1704,8 +1704,8 @@ get_relation_statistics_worker(List **stainfos, RelOptInfo *rel, info->inherit = dataForm->stxdinherit; info->rel = rel; info->kind = STATS_EXT_MCV; - info->keys = bms_copy(keys); info->exprs = exprs; + info->columns = columns; *stainfos = lappend(*stainfos, info); } @@ -1718,8 +1718,8 @@ get_relation_statistics_worker(List **stainfos, RelOptInfo *rel, info->inherit = dataForm->stxdinherit; info->rel = rel; info->kind = STATS_EXT_EXPRESSIONS; - info->keys = bms_copy(keys); info->exprs = exprs; + info->columns = columns; *stainfos = lappend(*stainfos, info); } @@ -1749,88 +1749,60 @@ get_relation_statistics(PlannerInfo *root, RelOptInfo *rel, foreach(l, statoidlist) { Oid statOid = lfirst_oid(l); - Form_pg_statistic_ext staForm; HeapTuple htup; - Bitmapset *keys = NULL; List *exprs = NIL; - int i; + Bitmapset *columns = NULL; htup = SearchSysCache1(STATEXTOID, ObjectIdGetDatum(statOid)); if (!HeapTupleIsValid(htup)) elog(ERROR, "cache lookup failed for statistics object %u", statOid); - staForm = (Form_pg_statistic_ext) GETSTRUCT(htup); - - /* - * First, build the array of columns covered. This is ultimately - * wasted if no stats within the object have actually been built, but - * it doesn't seem worth troubling over that case. - */ - for (i = 0; i < staForm->stxkeys.dim1; i++) - keys = bms_add_member(keys, staForm->stxkeys.values[i]); /* - * Preprocess expressions (if any). We read the expressions, fix the + * Preprocess the columns and expressions. We read them, fix the * varnos, and run them through eval_const_expressions. * * XXX We don't know yet if there are any data for this stats object, * with either stxdinherit value. But it's reasonable to assume there * is at least one of those, possibly both. So it's better to process - * keys and expressions here. + * columns and expressions here. */ - { - bool isnull; - Datum datum; - - /* decode expression (if any) */ - datum = SysCacheGetAttr(STATEXTOID, htup, - Anum_pg_statistic_ext_stxexprs, &isnull); + exprs = statext_get_stxexprs(htup, relation, &columns); - if (!isnull) - { - char *exprsString; - - exprsString = TextDatumGetCString(datum); - exprs = (List *) stringToNode(exprsString); - pfree(exprsString); - - /* Expand virtual generated columns in the expressions */ - exprs = (List *) expand_generated_columns_in_expr((Node *) exprs, relation, 1); - - /* - * Modify the copies we obtain from the relcache to have the - * correct varno for the parent relation, so that they match - * up correctly against qual clauses. - * - * This must be done before const-simplification because - * eval_const_expressions reduces NullTest for Vars based on - * varno. - */ - if (varno != 1) - ChangeVarNodes((Node *) exprs, 1, varno, 0); + /* + * Modify the copies we obtain from the relcache to have the correct + * varno for the parent relation, so that they match up correctly + * against qual clauses. + * + * This must be done before const-simplification because + * eval_const_expressions reduces NullTest for Vars based on varno. + */ + if (varno != 1) + ChangeVarNodes((Node *) exprs, 1, varno, 0); - /* - * Run the expressions through eval_const_expressions. This is - * not just an optimization, but is necessary, because the - * planner will be comparing them to similarly-processed qual - * clauses, and may fail to detect valid matches without this. - * We must not use canonicalize_qual, however, since these - * aren't qual expressions. - */ - exprs = (List *) eval_const_expressions(root, (Node *) exprs); + /* + * Run the columns and expressions through eval_const_expressions. + * This is not just an optimization, but is necessary, because the + * planner will be comparing them to similarly-processed qual clauses, + * and may fail to detect valid matches without this. We must not use + * canonicalize_qual, however, since these aren't qual expressions. + * Plain-column Vars are unaffected. statext_get_stxexprs() already + * const-folded these with a NULL root; we redo it here with the real + * root so the varno-dependent reductions noted above can apply. + */ + exprs = (List *) eval_const_expressions(root, (Node *) exprs); - /* May as well fix opfuncids too */ - fix_opfuncids((Node *) exprs); - } - } + /* May as well fix opfuncids too */ + fix_opfuncids((Node *) exprs); /* extract statistics for possible values of stxdinherit flag */ - get_relation_statistics_worker(&stainfos, rel, statOid, true, keys, exprs); + get_relation_statistics_worker(&stainfos, rel, statOid, true, + exprs, columns); - get_relation_statistics_worker(&stainfos, rel, statOid, false, keys, exprs); + get_relation_statistics_worker(&stainfos, rel, statOid, false, + exprs, columns); ReleaseSysCache(htup); - bms_free(keys); } list_free(statoidlist); diff --git a/src/backend/parser/parse_utilcmd.c b/src/backend/parser/parse_utilcmd.c index 583346bf531..76e85b8898c 100644 --- a/src/backend/parser/parse_utilcmd.c +++ b/src/backend/parser/parse_utilcmd.c @@ -2062,9 +2062,9 @@ generateClonedIndexStmt(RangeVar *heapRel, Relation source_idx, * extended statistic "source_statsid", for the rel identified by heapRel and * heapRelid. * - * stxkeys in the source statistic holds attribute numbers from the parent + * stxexprs in the source statistic holds Var nodes referencing the parent * relation. Those attnums, along with the attribute numbers referenced by - * Vars inside the expression tree, are remapped to the new relation's + * Vars inside complex expressions, are remapped to the new relation's * numbering according to attmap. */ static CreateStatsStmt * @@ -2072,15 +2072,16 @@ generateClonedExtStatsStmt(RangeVar *heapRel, Oid heapRelid, Oid source_statsid, const AttrMap *attmap) { HeapTuple ht_stats; - Form_pg_statistic_ext statsrec; CreateStatsStmt *stats; List *stat_types = NIL; List *def_names = NIL; - bool isnull; Datum datum; ArrayType *arr; char *enabled; int i; + ListCell *lc; + List *exprs = NIL; + char *exprsString; Assert(OidIsValid(heapRelid)); Assert(heapRel != NULL); @@ -2091,7 +2092,6 @@ generateClonedExtStatsStmt(RangeVar *heapRel, Oid heapRelid, ht_stats = SearchSysCache1(STATEXTOID, ObjectIdGetDatum(source_statsid)); if (!HeapTupleIsValid(ht_stats)) elog(ERROR, "cache lookup failed for statistics object %u", source_statsid); - statsrec = (Form_pg_statistic_ext) GETSTRUCT(ht_stats); /* Determine which statistics types exist */ datum = SysCacheGetAttrNotNull(STATEXTOID, ht_stats, @@ -2117,44 +2117,31 @@ generateClonedExtStatsStmt(RangeVar *heapRel, Oid heapRelid, elog(ERROR, "unrecognized statistics kind %c", enabled[i]); } - /* Determine which columns the statistics are on */ - for (i = 0; i < statsrec->stxkeys.dim1; i++) - { - StatsElem *selem = makeNode(StatsElem); - AttrNumber attnum = statsrec->stxkeys.values[i]; - - selem->name = - get_attname(heapRelid, attmap->attnums[attnum - 1], false); - selem->expr = NULL; - - def_names = lappend(def_names, selem); - } - /* - * Now handle expressions, if there are any. The order (with respect to - * regular attributes) does not really matter for extended stats, so we - * simply append them after simple column references. - * - * XXX Some places during build/estimation treat expressions as if they - * are before attributes, but for the CREATE command that's entirely - * irrelevant. + * Decode stxexprs to reconstruct the list of columns and expressions. + * Simple Var nodes represent plain column references; other nodes are + * complex expressions. */ - datum = SysCacheGetAttr(STATEXTOID, ht_stats, - Anum_pg_statistic_ext_stxexprs, &isnull); + datum = SysCacheGetAttrNotNull(STATEXTOID, ht_stats, + Anum_pg_statistic_ext_stxexprs); + exprsString = TextDatumGetCString(datum); + exprs = (List *) stringToNode(exprsString); - if (!isnull) + foreach(lc, exprs) { - ListCell *lc; - List *exprs = NIL; - char *exprsString; + Node *expr = (Node *) lfirst(lc); + StatsElem *selem = makeNode(StatsElem); - exprsString = TextDatumGetCString(datum); - exprs = (List *) stringToNode(exprsString); + if (IsA(expr, Var) && ((Var *) expr)->varattno > 0) + { + AttrNumber attnum = ((Var *) expr)->varattno; - foreach(lc, exprs) + selem->name = + get_attname(heapRelid, attmap->attnums[attnum - 1], false); + selem->expr = NULL; + } + else { - Node *expr = (Node *) lfirst(lc); - StatsElem *selem = makeNode(StatsElem); bool found_whole_row; /* Adjust Vars to match new table's column numbering */ @@ -2166,13 +2153,13 @@ generateClonedExtStatsStmt(RangeVar *heapRel, Oid heapRelid, selem->name = NULL; selem->expr = expr; - - def_names = lappend(def_names, selem); } - pfree(exprsString); + def_names = lappend(def_names, selem); } + pfree(exprsString); + /* finally, build the output node */ stats = makeNode(CreateStatsStmt); stats->defnames = NULL; diff --git a/src/backend/statistics/dependencies.c b/src/backend/statistics/dependencies.c index 612194193d9..492637fd82d 100644 --- a/src/backend/statistics/dependencies.c +++ b/src/backend/statistics/dependencies.c @@ -394,12 +394,19 @@ statext_dependencies_build(StatsBuildData *data) d = (MVDependency *) palloc0(offsetof(MVDependency, attributes) + k * sizeof(AttrNumber)); - /* copy the dependency (and keep the indexes into stxkeys) */ + /* copy the dependency */ d->degree = degree; d->nattributes = k; for (i = 0; i < k; i++) d->attributes[i] = data->attnums[dependency[i]]; + /* + * Order the first (k-1) attnums the way dump and restore needs + * (see compare_attnums); the last is the dependent attribute and + * stays last. + */ + qsort(d->attributes, k - 1, sizeof(AttrNumber), compare_attnums); + /* initialize the list of dependencies */ if (dependencies == NULL) { @@ -597,13 +604,13 @@ statext_dependencies_free(MVDependencies *dependencies) * attributes list correspond to attnums/expressions defined by the * extended statistics object. * - * Positive attnums are attributes which must be found in the stxkeys, while - * negative attnums correspond to an expression number, no attribute number - * can be below (0 - numexprs). + * Positive attnums correspond to table columns (excluding virtual generated + * columns), while negative attnums correspond to expressions. No attribute + * number can be below (0 - numexprs). */ bool statext_dependencies_validate(const MVDependencies *dependencies, - const int2vector *stxkeys, + const Bitmapset *keys, int numexprs, int elevel) { int attnum_expr_lowbound = 0 - numexprs; @@ -624,15 +631,8 @@ statext_dependencies_validate(const MVDependencies *dependencies, if (attnum > 0) { - /* attribute number in stxkeys */ - for (int k = 0; k < stxkeys->dim1; k++) - { - if (attnum == stxkeys->values[k]) - { - ok = true; - break; - } - } + /* attribute number in keys */ + ok = bms_is_member(attnum, keys); } else if ((attnum < 0) && (attnum >= attnum_expr_lowbound)) { @@ -1309,6 +1309,9 @@ dependency_is_compatible_expression(Node *clause, Index relid, List *statlist, N { Node *stat_expr = (Node *) lfirst(lc2); + if (bms_is_member(foreach_current_index(lc2), info->columns)) + continue; + if (equal(clause_expr, stat_expr)) { *stat_expr_p = stat_expr; @@ -1536,7 +1539,7 @@ dependencies_clauselist_selectivity(PlannerInfo *root, StatisticExtInfo *stat = (StatisticExtInfo *) lfirst(l); int nmatched; int nexprs; - int k; + ListCell *lc_col; MVDependencies *deps; /* skip statistics that are not of the correct type */ @@ -1548,21 +1551,22 @@ dependencies_clauselist_selectivity(PlannerInfo *root, continue; /* - * Count matching attributes - we have to undo the attnum offsets. The - * input attribute numbers are not offset (expressions are not - * included in stat->keys, so it's not necessary). But we need to - * offset it before checking against clauses_attnums. + * Count matching attributes. We walk the covered plain columns + * (expression entries are skipped) and offset each attnum before + * checking it against clauses_attnums. */ nmatched = 0; - k = -1; - while ((k = bms_next_member(stat->keys, k)) >= 0) + foreach(lc_col, stat->exprs) { - AttrNumber attnum = (AttrNumber) k; + Node *node = (Node *) lfirst(lc_col); + AttrNumber attnum; - /* skip expressions */ - if (!AttrNumberIsForUserDefinedAttr(attnum)) + /* only plain columns are counted here */ + if (!bms_is_member(foreach_current_index(lc_col), stat->columns)) continue; + attnum = ((Var *) node)->varattno; + /* apply the same offset as above */ attnum += attnum_offset; @@ -1580,6 +1584,9 @@ dependencies_clauselist_selectivity(PlannerInfo *root, { Node *stat_expr = (Node *) lfirst(lc); + if (bms_is_member(foreach_current_index(lc), stat->columns)) + continue; + /* try to match it */ if (equal(stat_expr, unique_exprs[i])) nexprs++; @@ -1621,7 +1628,7 @@ dependencies_clauselist_selectivity(PlannerInfo *root, * it in a cheaper way (if there are no expr clauses, we can just * discard all negative attnums without any lookups). */ - if (unique_exprs_cnt > 0 || stat->exprs != NIL) + if (unique_exprs_cnt > 0 || stat_num_expressions(stat) > 0) { uint32 ndeps = 0; @@ -1676,9 +1683,9 @@ dependencies_clauselist_selectivity(PlannerInfo *root, idx = -(1 + attnum); /* Is the expression index is valid? */ - Assert((idx >= 0) && (idx < list_length(stat->exprs))); + Assert((idx >= 0) && (idx < stat_num_expressions(stat))); - expr = (Node *) list_nth(stat->exprs, idx); + expr = stat_nth_expression(stat, idx); /* try to find the expression in the unique list */ for (int m = 0; m < unique_exprs_cnt; m++) diff --git a/src/backend/statistics/extended_stats.c b/src/backend/statistics/extended_stats.c index c8140624e06..3080af0916a 100644 --- a/src/backend/statistics/extended_stats.c +++ b/src/backend/statistics/extended_stats.c @@ -67,15 +67,16 @@ typedef struct StatExtEntry Oid statOid; /* OID of pg_statistic_ext entry */ char *schema; /* statistics object's schema */ char *name; /* statistics object's name */ - Bitmapset *columns; /* attribute numbers covered by the object */ List *types; /* 'char' list of enabled statistics kinds */ int stattarget; /* statistics target (-1 for default) */ - List *exprs; /* expressions */ + List *exprs; /* all columns and expressions in + * user-declared order, mirroring stxexprs */ + Bitmapset *columns; /* positions in exprs of the plain columns */ } StatExtEntry; static List *fetch_statentries_for_relation(Relation pg_statext, Relation rel); -static VacAttrStats **lookup_var_attr_stats(Bitmapset *attrs, List *exprs, +static VacAttrStats **lookup_var_attr_stats(StatExtEntry *stat, int nvacatts, VacAttrStats **vacatts); static void statext_store(Oid statOid, bool inh, MVNDistinct *ndistinct, MVDependencies *dependencies, @@ -178,8 +179,7 @@ BuildRelationExtStatistics(Relation onerel, bool inh, double totalrows, * Check if we can build these stats based on the column analyzed. If * not, report this fact (except in autovacuum) and move on. */ - stats = lookup_var_attr_stats(stat->columns, stat->exprs, - natts, vacattrstats); + stats = lookup_var_attr_stats(stat, natts, vacattrstats); if (!stats) { report_ext_stats_not_computed(onerel, stat); @@ -188,7 +188,7 @@ BuildRelationExtStatistics(Relation onerel, bool inh, double totalrows, /* compute statistics target for this statistics object */ stattarget = statext_compute_stattarget(stat->stattarget, - bms_num_members(stat->columns), + list_length(stat->exprs), stats); /* @@ -222,23 +222,45 @@ BuildRelationExtStatistics(Relation onerel, bool inh, double totalrows, { AnlExprData *exprdata; int nexprs; + List *cmplxexprs = NIL; + ListCell *lc3; - /* should not happen, thanks to checks when defining stats */ - if (!stat->exprs) - elog(ERROR, "requested expression stats, but there are no expressions"); - - exprdata = build_expr_data(stat->exprs, stattarget); - if (!exprdata) + /* + * stat->exprs holds all columns and expressions in declared + * order; expression stats cover only the expressions, so + * filter out the plain columns. + */ + foreach(lc3, stat->exprs) { - report_ext_stats_not_computed(onerel, stat); - data = NULL; - break; - } - nexprs = list_length(stat->exprs); + Node *node = (Node *) lfirst(lc3); + + if (bms_is_member(foreach_current_index(lc3), stat->columns)) + continue; - compute_expr_stats(onerel, exprdata, nexprs, rows, numrows); + cmplxexprs = lappend(cmplxexprs, node); + } - exprstats = serialize_expr_stats(exprdata, nexprs); + /* + * Build per-expression stats only if there are expressions; + * otherwise there is nothing to compute, and stxdexpr is left + * NULL (like any other kind whose build finds nothing; see + * statext_store). + */ + if (cmplxexprs != NIL) + { + exprdata = build_expr_data(cmplxexprs, stattarget); + if (!exprdata) + { + report_ext_stats_not_computed(onerel, stat); + data = NULL; + break; + } + nexprs = list_length(cmplxexprs); + + compute_expr_stats(onerel, exprdata, nexprs, rows, numrows); + + exprstats = serialize_expr_stats(exprdata, nexprs); + } } } @@ -340,15 +362,14 @@ ComputeExtStatisticsRows(Relation onerel, StatExtEntry *stat = (StatExtEntry *) lfirst(lc); int stattarget; VacAttrStats **stats; - int nattrs = bms_num_members(stat->columns); + int nattrs = list_length(stat->exprs); /* * Check if we can build this statistics object based on the columns * analyzed. If not, ignore it (don't report anything, we'll do that * during the actual build BuildRelationExtStatistics). */ - stats = lookup_var_attr_stats(stat->columns, stat->exprs, - natts, vacattrstats); + stats = lookup_var_attr_stats(stat, natts, vacattrstats); if (!stats) continue; @@ -466,6 +487,171 @@ statext_is_kind_built(HeapTuple htup, char type) return !heap_attisnull(htup, attnum, NULL); } +/* + * statext_get_stxexprs + * Decode the stxexprs field of a pg_statistic_ext tuple into the full + * list of columns and expressions, in user-declared order. + * + * Deserializes the expression list, expands virtual generated columns, and + * const-folds the whole list (as RelationGetIndexExpressions does). The + * returned list contains all of them, both simple Var references (plain + * columns) and complex expressions, in the order they were declared in + * CREATE STATISTICS, mirroring the catalog stxexprs. + * + * Const-folding the whole list (rather than only the complex expressions) is + * harmless, because plain Var nodes are unaffected by eval_const_expressions. + * + * If columns is not NULL, *columns is set to the positions of the entries + * that are plain columns. This is decided on the entries as written, before + * expansion and const-folding, which can turn an expression into a bare Var + * (e.g. "CASE WHEN true THEN a END", or a virtual generated column defined as + * "a"); such an entry remains an expression. + */ +List * +statext_get_stxexprs(HeapTuple htup, Relation rel, Bitmapset **columns) +{ + Datum datum; + char *exprsString; + List *allexprs; + + datum = SysCacheGetAttrNotNull(STATEXTOID, htup, + Anum_pg_statistic_ext_stxexprs); + exprsString = TextDatumGetCString(datum); + allexprs = (List *) stringToNode(exprsString); + pfree(exprsString); + + if (columns != NULL) + *columns = statext_column_entries(allexprs, rel); + + /* Expand virtual generated columns in the expressions */ + allexprs = (List *) expand_generated_columns_in_expr((Node *) allexprs, rel, 1); + + /* + * Run the expressions through eval_const_expressions. This is not just an + * optimization, but is necessary, because the planner will be comparing + * them to similarly-processed qual clauses, and may fail to detect valid + * matches without this. We must not use canonicalize_qual, however, + * since these aren't qual expressions. + */ + allexprs = (List *) eval_const_expressions(NULL, (Node *) allexprs); + + /* May as well fix opfuncids too */ + fix_opfuncids((Node *) allexprs); + + return allexprs; +} + +/* + * statext_column_entries + * Positions of the entries of a statistics object that are plain columns. + * + * exprs must be the entries as stored in stxexprs, not yet expanded or + * const-folded. An entry is a plain column if it is a Var on a user column + * that is not a virtual generated column; everything else is an expression. + * Only the anchor relation (varno 1) can have virtual generated columns here, + * as join statistics do not allow them. + */ +Bitmapset * +statext_column_entries(List *exprs, Relation rel) +{ + Bitmapset *columns = NULL; + ListCell *lc; + + foreach(lc, exprs) + { + Node *node = (Node *) lfirst(lc); + Var *var; + + if (!IsA(node, Var)) + continue; + + var = (Var *) node; + if (!AttrNumberIsForUserDefinedAttr(var->varattno)) + continue; + + if (var->varno == 1 && + TupleDescAttr(RelationGetDescr(rel), var->varattno - 1)->attgenerated == + ATTRIBUTE_GENERATED_VIRTUAL) + continue; + + columns = bms_add_member(columns, foreach_current_index(lc)); + } + + return columns; +} + +/* + * stat_covers_attnum + * Is attnum one of the statistics object's columns? + */ +bool +stat_covers_attnum(StatisticExtInfo *stat, AttrNumber attnum) +{ + ListCell *lc; + + foreach(lc, stat->exprs) + { + Node *node = (Node *) lfirst(lc); + + if (bms_is_member(foreach_current_index(lc), stat->columns) && + ((Var *) node)->varattno == attnum) + return true; + } + return false; +} + +/* + * stat_covers_attnums + * Does the statistics object cover every attnum in the set? + * An empty set is trivially covered. + */ +static bool +stat_covers_attnums(StatisticExtInfo *stat, Bitmapset *attnums) +{ + int k = -1; + + while ((k = bms_next_member(attnums, k)) >= 0) + { + if (!stat_covers_attnum(stat, (AttrNumber) k)) + return false; + } + return true; +} + +/* + * stat_num_expressions + * Number of expressions (non-column entries) of the statistics object. + */ +int +stat_num_expressions(StatisticExtInfo *stat) +{ + return list_length(stat->exprs) - bms_num_members(stat->columns); +} + +/* + * stat_nth_expression + * Return the n-th expression (non-column entry), in the per-expression + * ordering used by the stored stxdexpr statistics. + */ +Node * +stat_nth_expression(StatisticExtInfo *stat, int n) +{ + ListCell *lc; + int idx = 0; + + foreach(lc, stat->exprs) + { + Node *node = (Node *) lfirst(lc); + + if (bms_is_member(foreach_current_index(lc), stat->columns)) + continue; + if (idx == n) + return node; + idx++; + } + return NULL; +} + /* * Return a list (of StatExtEntry) of statistics objects for the given relation. */ @@ -499,18 +685,12 @@ fetch_statentries_for_relation(Relation pg_statext, Relation rel) ArrayType *arr; char *enabled; Form_pg_statistic_ext staForm; - List *exprs = NIL; entry = palloc0_object(StatExtEntry); staForm = (Form_pg_statistic_ext) GETSTRUCT(htup); entry->statOid = staForm->oid; entry->schema = get_namespace_name(staForm->stxnamespace); entry->name = pstrdup(NameStr(staForm->stxname)); - for (i = 0; i < staForm->stxkeys.dim1; i++) - { - entry->columns = bms_add_member(entry->columns, - staForm->stxkeys.values[i]); - } datum = SysCacheGetAttr(STATEXTOID, htup, Anum_pg_statistic_ext_stxstattarget, &isnull); entry->stattarget = isnull ? -1 : DatumGetInt16(datum); @@ -533,37 +713,11 @@ fetch_statentries_for_relation(Relation pg_statext, Relation rel) entry->types = lappend_int(entry->types, (int) enabled[i]); } - /* decode expression (if any) */ - datum = SysCacheGetAttr(STATEXTOID, htup, - Anum_pg_statistic_ext_stxexprs, &isnull); - - if (!isnull) - { - char *exprsString; - - exprsString = TextDatumGetCString(datum); - exprs = (List *) stringToNode(exprsString); - - pfree(exprsString); - - /* Expand virtual generated columns in the expressions */ - exprs = (List *) expand_generated_columns_in_expr((Node *) exprs, rel, 1); - - /* - * Run the expressions through eval_const_expressions. This is not - * just an optimization, but is necessary, because the planner - * will be comparing them to similarly-processed qual clauses, and - * may fail to detect valid matches without this. We must not use - * canonicalize_qual, however, since these aren't qual - * expressions. - */ - exprs = (List *) eval_const_expressions(NULL, (Node *) exprs); - - /* May as well fix opfuncids too */ - fix_opfuncids((Node *) exprs); - } - - entry->exprs = exprs; + /* + * Decode stxexprs into all columns and expressions, in declared + * order. + */ + entry->exprs = statext_get_stxexprs(htup, rel, &entry->columns); result = lappend(result, entry); } @@ -739,79 +893,78 @@ examine_expression(Node *expr, int stattarget) /* * Using 'vacatts' of size 'nvacatts' as input data, return a newly-built - * VacAttrStats array which includes only the items corresponding to - * attributes indicated by 'attrs'. If we don't have all of the per-column - * stats available to compute the extended stats, then we return NULL to - * indicate to the caller that the stats should not be built. + * VacAttrStats array which includes only the items corresponding to the + * columns and expressions in 'exprs', in the same order. If we don't have all + * of the per-column stats available to compute the extended stats, then we + * return NULL to indicate to the caller that the stats should not be built. */ static VacAttrStats ** -lookup_var_attr_stats(Bitmapset *attrs, List *exprs, - int nvacatts, VacAttrStats **vacatts) +lookup_var_attr_stats(StatExtEntry *stat, int nvacatts, VacAttrStats **vacatts) { int i = 0; - int x = -1; int natts; VacAttrStats **stats; ListCell *lc; - natts = bms_num_members(attrs) + list_length(exprs); + natts = list_length(stat->exprs); stats = palloc_array(VacAttrStats *, natts); - /* lookup VacAttrStats info for the requested columns (same attnum) */ - while ((x = bms_next_member(attrs, x)) >= 0) + foreach(lc, stat->exprs) { - int j; + Node *node = (Node *) lfirst(lc); - stats[i] = NULL; - for (j = 0; j < nvacatts; j++) + if (bms_is_member(foreach_current_index(lc), stat->columns)) { - if (x == vacatts[j]->tupattnum) + AttrNumber attno = ((Var *) node)->varattno; + int j; + + /* lookup VacAttrStats info for a plain column (same attnum) */ + stats[i] = NULL; + for (j = 0; j < nvacatts; j++) { - stats[i] = vacatts[j]; - break; + if (attno == vacatts[j]->tupattnum) + { + stats[i] = vacatts[j]; + break; + } } - } - if (!stats[i]) + if (!stats[i]) + { + /* + * Looks like stats were not gathered for one of the columns + * required. We'll be unable to build the extended stats + * without this column. + */ + pfree(stats); + return NULL; + } + } + else { + /* an expression */ + stats[i] = examine_attribute(node); + /* - * Looks like stats were not gathered for one of the columns - * required. We'll be unable to build the extended stats without - * this column. + * If the expression has been found as non-analyzable, give up. We + * will not be able to build extended stats with it. */ - pfree(stats); - return NULL; - } - - i++; - } - - /* also add info for expressions */ - foreach(lc, exprs) - { - Node *expr = (Node *) lfirst(lc); - - stats[i] = examine_attribute(expr); + if (stats[i] == NULL) + { + pfree(stats); + return NULL; + } - /* - * If the expression has been found as non-analyzable, give up. We - * will not be able to build extended stats with it. - */ - if (stats[i] == NULL) - { - pfree(stats); - return NULL; + /* + * XXX We need tuple descriptor later, and we just grab it from + * stats[0]->tupDesc (see e.g. statext_mcv_build). But as coded + * examine_attribute does not set that, so just grab it from the + * first vacatts element. + */ + stats[i]->tupDesc = vacatts[0]->tupDesc; } - /* - * XXX We need tuple descriptor later, and we just grab it from - * stats[0]->tupDesc (see e.g. statext_mcv_build). But as coded - * examine_attribute does not set that, so just grab it from the first - * vacatts element. - */ - stats[i]->tupDesc = vacatts[0]->tupDesc; - i++; } @@ -995,49 +1148,33 @@ compare_datums_simple(Datum a, Datum b, SortSupport ssup) } /* - * build_attnums_array - * Transforms a bitmap into an array of AttrNumber values. + * compare_attnums + * qsort comparator that orders attribute numbers with columns (positive + * attnums) before expressions (negative attnums), columns in ascending + * and expressions in descending order. * - * This is used for extended statistics only, so all the attributes must be - * user-defined. That means offsetting by FirstLowInvalidHeapAttributeNumber - * is not necessary here (and when querying the bitmap). + * The order of the attribute numbers within an ndistinct item or a dependency + * does not matter for estimation, which looks them up by value. It matters + * only for pg_dump and pg_upgrade: they store the statistics as text and load + * it back through the pg_ndistinct and pg_dependencies input functions, which + * accept the attribute numbers only in this order. */ -AttrNumber * -build_attnums_array(Bitmapset *attrs, int nexprs, int *numattrs) +int +compare_attnums(const void *a, const void *b) { - int i, - j; - AttrNumber *attnums; - int num = bms_num_members(attrs); - - if (numattrs) - *numattrs = num; - - /* build attnums from the bitmapset */ - attnums = palloc_array(AttrNumber, num); - i = 0; - j = -1; - while ((j = bms_next_member(attrs, j)) >= 0) - { - int attnum = (j - nexprs); - - /* - * Make sure the bitmap contains only user-defined attributes. As - * bitmaps can't contain negative values, this can be violated in two - * ways. Firstly, the bitmap might contain 0 as a member, and secondly - * the integer value might be larger than MaxAttrNumber. - */ - Assert(AttributeNumberIsValid(attnum)); - Assert(attnum <= MaxAttrNumber); - Assert(attnum >= (-nexprs)); - - attnums[i++] = (AttrNumber) attnum; - - /* protect against overflows */ - Assert(i <= num); - } - - return attnums; + AttrNumber x = *(const AttrNumber *) a; + AttrNumber y = *(const AttrNumber *) b; + + /* columns sort ahead of expressions */ + if ((x > 0) && (y < 0)) + return -1; + if ((x < 0) && (y > 0)) + return 1; + + /* columns ascending, expressions descending */ + if (x > 0) + return x - y; + return y - x; } /* @@ -1214,6 +1351,10 @@ stat_find_expression(StatisticExtInfo *stat, Node *expr) { Node *stat_expr = (Node *) lfirst(lc); + /* columns have no per-expression stats, so count expressions only */ + if (bms_is_member(foreach_current_index(lc), stat->columns)) + continue; + if (equal(stat_expr, expr)) return idx; idx++; @@ -1315,7 +1456,7 @@ choose_best_statistics(List *stats, char requiredkind, bool inh, continue; /* ignore clauses that are not covered by this object */ - if (!bms_is_subset(clause_attnums[i], info->keys) || + if (!stat_covers_attnums(info, clause_attnums[i]) || !stat_covers_expressions(info, clause_exprs[i], &expr_idxs)) continue; @@ -1333,7 +1474,7 @@ choose_best_statistics(List *stats, char requiredkind, bool inh, * save the actual number of keys in the stats so that we can choose * the narrowest stats with the most matching keys. */ - numkeys = bms_num_members(info->keys) + list_length(info->exprs); + numkeys = list_length(info->exprs); /* * Use this object when it increases the number of matched attributes @@ -1849,7 +1990,7 @@ statext_mcv_clauselist_selectivity(PlannerInfo *root, List *clauses, int varReli * This also eliminates already estimated clauses - both those * estimated before and during applying extended statistics. * - * XXX This check is needed because both bms_is_subset and + * XXX This check is needed because both stat_covers_attnums and * stat_covers_expressions return true for empty attnums and * expressions. */ @@ -1864,7 +2005,7 @@ statext_mcv_clauselist_selectivity(PlannerInfo *root, List *clauses, int varReli * We need to check both attributes and expressions, and reject if * either is not covered. */ - if (!bms_is_subset(list_attnums[listidx], stat->keys) || + if (!stat_covers_attnums(stat, list_attnums[listidx]) || !stat_covers_expressions(stat, list_exprs[listidx], NULL)) continue; @@ -2462,7 +2603,8 @@ make_build_data(Relation rel, StatExtEntry *stat, int numrows, HeapTuple *rows, EState *estate; ExprContext *econtext; List *exprstates = NIL; - int nkeys = bms_num_members(stat->columns) + list_length(stat->exprs); + List *cmplxexprs = NIL; + int nkeys = list_length(stat->exprs); ListCell *lc; /* allocate everything as a single chunk, so we can free it easily */ @@ -2515,24 +2657,27 @@ make_build_data(Relation rel, StatExtEntry *stat, int numrows, HeapTuple *rows, result->nattnums = nkeys; result->numrows = numrows; - /* fill the attribute info - first attributes, then expressions */ + /* fill the attribute info for each column and expression */ idx = 0; k = -1; - while ((k = bms_next_member(stat->columns, k)) >= 0) + foreach(lc, stat->exprs) { - result->attnums[idx] = k; - result->stats[idx] = stats[idx]; + Node *node = (Node *) lfirst(lc); - idx++; - } + if (bms_is_member(foreach_current_index(lc), stat->columns)) + { + result->attnums[idx] = ((Var *) node)->varattno; + result->stats[idx] = stats[idx]; + } + else + { + result->attnums[idx] = k; + result->stats[idx] = examine_expression(node, stattarget); - k = -1; - foreach(lc, stat->exprs) - { - Node *expr = (Node *) lfirst(lc); + cmplxexprs = lappend(cmplxexprs, node); - result->attnums[idx] = k; - result->stats[idx] = examine_expression(expr, stattarget); + k--; + } if (result->stats[idx] == NULL) { @@ -2541,22 +2686,6 @@ make_build_data(Relation rel, StatExtEntry *stat, int numrows, HeapTuple *rows, } idx++; - k--; - } - - /* first extract values for all the regular attributes */ - for (i = 0; i < numrows; i++) - { - idx = 0; - k = -1; - while ((k = bms_next_member(stat->columns, k)) >= 0) - { - result->values[idx][i] = heap_getattr(rows[i], k, - result->stats[idx]->tupDesc, - &result->nulls[idx][i]); - - idx++; - } } /* Need an EState for evaluation expressions. */ @@ -2570,11 +2699,13 @@ make_build_data(Relation rel, StatExtEntry *stat, int numrows, HeapTuple *rows, /* Arrange for econtext's scan tuple to be the tuple under test */ econtext->ecxt_scantuple = slot; - /* Set up expression evaluation state */ - exprstates = ExecPrepareExprList(stat->exprs, estate); + /* Set up expression evaluation state, for the complex expressions only */ + exprstates = ExecPrepareExprList(cmplxexprs, estate); for (i = 0; i < numrows; i++) { + ListCell *lc_state = list_head(exprstates); + /* * Reset the per-tuple context each time, to reclaim any cruft left * behind by evaluating the statistics object expressions. @@ -2584,32 +2715,44 @@ make_build_data(Relation rel, StatExtEntry *stat, int numrows, HeapTuple *rows, /* Set up for expression evaluation */ ExecStoreHeapTuple(rows[i], slot, false); - idx = bms_num_members(stat->columns); - foreach(lc, exprstates) + idx = 0; + foreach(lc, stat->exprs) { - Datum datum; - bool isnull; - ExprState *exprstate = (ExprState *) lfirst(lc); + Node *node = (Node *) lfirst(lc); - /* - * Avoid accumulating per-row evaluation memory in the - * long-lived build context. - */ - datum = ExecEvalExprSwitchContext(exprstate, - GetPerTupleExprContext(estate), - &isnull); - if (isnull) + if (bms_is_member(foreach_current_index(lc), stat->columns)) { - result->values[idx][i] = (Datum) 0; - result->nulls[idx][i] = true; + result->values[idx][i] = + heap_getattr(rows[i], ((Var *) node)->varattno, + result->stats[idx]->tupDesc, + &result->nulls[idx][i]); } else { - result->values[idx][i] = - datumCopy(datum, - result->stats[idx]->attrtype->typbyval, - result->stats[idx]->attrtype->typlen); - result->nulls[idx][i] = false; + Datum datum; + bool isnull; + ExprState *exprstate = (ExprState *) lfirst(lc_state); + + lc_state = lnext(exprstates, lc_state); + + /* + * Avoid accumulating per-row evaluation memory in the + * long-lived build context. + */ + datum = ExecEvalExprSwitchContext(exprstate, econtext, &isnull); + if (isnull) + { + result->values[idx][i] = (Datum) 0; + result->nulls[idx][i] = true; + } + else + { + result->values[idx][i] = + datumCopy(datum, + result->stats[idx]->attrtype->typbyval, + result->stats[idx]->attrtype->typlen); + result->nulls[idx][i] = false; + } } idx++; diff --git a/src/backend/statistics/extended_stats_funcs.c b/src/backend/statistics/extended_stats_funcs.c index e8bdcf10a61..3f010533e25 100644 --- a/src/backend/statistics/extended_stats_funcs.c +++ b/src/backend/statistics/extended_stats_funcs.c @@ -18,6 +18,7 @@ #include "access/heapam.h" #include "access/relation.h" +#include "access/table.h" #include "catalog/indexing.h" #include "catalog/namespace.h" #include "catalog/pg_collation_d.h" @@ -147,10 +148,8 @@ static void upsert_pg_statistic_ext_data(const Datum *values, static bool check_mcvlist_array(const ArrayType *arr, int argindex, int required_ndims, int mcv_length); -static Datum import_expressions(Relation pgsd, int numexprs, - Oid *atttypids, int32 *atttypmods, - Oid *atttypcolls, Jsonb *exprs_jsonb, - bool *exprs_is_perfect); +static Datum import_expressions(Relation pgsd, List *exprnodes, + Jsonb *exprs_jsonb, bool *exprs_is_perfect); static Datum import_mcv(const ArrayType *mcv_arr, const ArrayType *freqs_arr, const ArrayType *base_freqs_arr, @@ -333,10 +332,11 @@ extended_statistics_update(FunctionCallInfo fcinfo) bool nulls[Natts_pg_statistic_ext_data] = {0}; bool replaces[Natts_pg_statistic_ext_data] = {0}; bool success = true; - Datum exprdatum; - bool isnull; + Relation rel; List *exprs = NIL; - int numattnums = 0; + Bitmapset *columns = NULL; + Bitmapset *keys = NULL; + ListCell *lc; int numexprs = 0; int numattrs = 0; @@ -344,6 +344,9 @@ extended_statistics_update(FunctionCallInfo fcinfo) Oid *atttypids = NULL; int32 *atttypmods = NULL; Oid *atttypcolls = NULL; + + /* the expressions (declared order), for import_expressions */ + List *exprnodes = NIL; Oid relid; Oid locked_table = InvalidOid; @@ -451,40 +454,25 @@ extended_statistics_update(FunctionCallInfo fcinfo) /* Find out what extended statistics kinds we should expect. */ expand_stxkind(tup, &enabled); - numattnums = stxform->stxkeys.dim1; - - /* decode expression (if any) */ - exprdatum = SysCacheGetAttr(STATEXTOID, - tup, - Anum_pg_statistic_ext_stxexprs, - &isnull); - if (!isnull) - { - char *s; - s = TextDatumGetCString(exprdatum); - exprs = (List *) stringToNode(s); - pfree(s); - - /* - * Run the expressions through eval_const_expressions(). This is not - * just an optimization, but is necessary, because the planner will be - * comparing them to similarly-processed qual clauses, and may fail to - * detect valid matches without this. - * - * We must not use canonicalize_qual(), however, since these are not - * qual expressions. - */ - exprs = (List *) eval_const_expressions(NULL, (Node *) exprs); + /* + * Decode stxexprs, and collect the column attnums and expression count + * needed to validate the ndistinct and dependencies inputs. + */ + rel = table_open(relid, NoLock); + exprs = statext_get_stxexprs(tup, rel, &columns); + table_close(rel, NoLock); - /* May as well fix opfuncids too */ - fix_opfuncids((Node *) exprs); + foreach(lc, exprs) + { + Node *node = (Node *) lfirst(lc); - /* Compute the number of expression, for input validation. */ - numexprs = list_length(exprs); + if (bms_is_member(foreach_current_index(lc), columns)) + keys = bms_add_member(keys, ((Var *) node)->varattno); + else + numexprs++; } - - numattrs = numattnums + numexprs; + numattrs = list_length(exprs); /* * If the object cannot support ndistinct, we should not have data for it. @@ -592,54 +580,53 @@ extended_statistics_update(FunctionCallInfo fcinfo) */ if (has.mcv || has.expressions) { + int idx; + atttypids = palloc0_array(Oid, numattrs); atttypmods = palloc0_array(int32, numattrs); atttypcolls = palloc0_array(Oid, numattrs); /* - * The leading stxkeys are attribute numbers up through numattnums. - * These keys must be in ascending AttrNumber order, but we do not - * rely on that. + * Get the type info for each dimension, in the same declared order as + * the stored MCV values. import_expressions() needs only the + * expressions, so collect those nodes as we go. */ - for (int i = 0; i < numattnums; i++) + idx = 0; + foreach(lc, exprs) { - AttrNumber attnum = stxform->stxkeys.values[i]; - HeapTuple atup = SearchSysCache2(ATTNUM, - ObjectIdGetDatum(relid), - Int16GetDatum(attnum)); - - Form_pg_attribute attr; + Node *node = (Node *) lfirst(lc); - /* Attribute not found */ - if (!HeapTupleIsValid(atup)) - elog(ERROR, "stxkeys references nonexistent attnum %d", attnum); + if (bms_is_member(foreach_current_index(lc), columns)) + { + AttrNumber attnum = ((Var *) node)->varattno; + HeapTuple atup = SearchSysCache2(ATTNUM, + ObjectIdGetDatum(relid), + Int16GetDatum(attnum)); + Form_pg_attribute attr; - attr = (Form_pg_attribute) GETSTRUCT(atup); + /* Attribute not found */ + if (!HeapTupleIsValid(atup)) + elog(ERROR, "stxexprs references nonexistent attnum %d", attnum); - if (attr->attisdropped) - elog(ERROR, "stxkeys references dropped attnum %d", attnum); + attr = (Form_pg_attribute) GETSTRUCT(atup); - atttypids[i] = attr->atttypid; - atttypmods[i] = attr->atttypmod; - atttypcolls[i] = attr->attcollation; - ReleaseSysCache(atup); - } + if (attr->attisdropped) + elog(ERROR, "stxexprs references dropped attnum %d", attnum); - /* - * After all the positive number attnums in stxkeys come the negative - * numbers (if any) which represent expressions in the order that they - * appear in stxdexpr. Because the expressions are always - * monotonically decreasing from -1, there is no point in looking at - * the values in stxkeys, it's enough to know how many of them there - * are. - */ - for (int i = numattnums; i < numattrs; i++) - { - Node *expr = list_nth(exprs, i - numattnums); + atttypids[idx] = attr->atttypid; + atttypmods[idx] = attr->atttypmod; + atttypcolls[idx] = attr->attcollation; + ReleaseSysCache(atup); + } + else + { + atttypids[idx] = exprType(node); + atttypmods[idx] = exprTypmod(node); + atttypcolls[idx] = exprCollation(node); - atttypids[i] = exprType(expr); - atttypmods[i] = exprTypmod(expr); - atttypcolls[i] = exprCollation(expr); + exprnodes = lappend(exprnodes, node); + } + idx++; } } @@ -671,7 +658,7 @@ extended_statistics_update(FunctionCallInfo fcinfo) bytea *data = DatumGetByteaPP(ndistinct_datum); MVNDistinct *ndistinct = statext_ndistinct_deserialize(data); - if (statext_ndistinct_validate(ndistinct, &stxform->stxkeys, + if (statext_ndistinct_validate(ndistinct, keys, numexprs, WARNING)) { values[Anum_pg_statistic_ext_data_stxdndistinct - 1] = ndistinct_datum; @@ -690,7 +677,7 @@ extended_statistics_update(FunctionCallInfo fcinfo) bytea *data = DatumGetByteaPP(dependencies_datum); MVDependencies *dependencies = statext_dependencies_deserialize(data); - if (statext_dependencies_validate(dependencies, &stxform->stxkeys, + if (statext_dependencies_validate(dependencies, keys, numexprs, WARNING)) { values[Anum_pg_statistic_ext_data_stxddependencies - 1] = dependencies_datum; @@ -733,18 +720,7 @@ extended_statistics_update(FunctionCallInfo fcinfo) pgsd = table_open(StatisticRelationId, RowExclusiveLock); - /* - * Generate the expressions array. - * - * The atttypids, atttypmods, and atttypcolls arrays have all the - * regular attributes listed first, so we can pass those arrays with a - * start point after the last regular attribute. There are numexprs - * elements remaining. - */ - datum = import_expressions(pgsd, numexprs, - &atttypids[numattnums], - &atttypmods[numattnums], - &atttypcolls[numattnums], + datum = import_expressions(pgsd, exprnodes, PG_GETARG_JSONB_P(EXPRESSIONS_ARG), &ok); @@ -1585,11 +1561,10 @@ pg_statistic_error: * This datum is needed to fill out a complete pg_statistic_ext_data tuple. */ static Datum -import_expressions(Relation pgsd, int numexprs, - Oid *atttypids, int32 *atttypmods, - Oid *atttypcolls, Jsonb *exprs_jsonb, - bool *exprs_is_perfect) +import_expressions(Relation pgsd, List *exprnodes, + Jsonb *exprs_jsonb, bool *exprs_is_perfect) { + int numexprs = list_length(exprnodes); const char *argname = extarginfo[EXPRESSIONS_ARG].argname; Oid pgstypoid = get_rel_type_id(StatisticRelationId); ArrayBuildState *astate = NULL; @@ -1651,12 +1626,13 @@ import_expressions(Relation pgsd, int numexprs, case jbvBinary: { bool sta_ok = false; + Node *node = (Node *) list_nth(exprnodes, i); /* a real stats object */ pgstdat = import_pg_statistic(pgsd, elem->val.binary.data, exprattnum, &array_in_fn, - atttypids[i], atttypmods[i], - atttypcolls[i], &sta_ok); + exprType(node), exprTypmod(node), + exprCollation(node), &sta_ok); /* * If some incorrect data has been found, assign NULL for diff --git a/src/backend/statistics/mcv.c b/src/backend/statistics/mcv.c index 10a91c97aeb..e0e329e4918 100644 --- a/src/backend/statistics/mcv.c +++ b/src/backend/statistics/mcv.c @@ -1529,48 +1529,37 @@ pg_mcv_list_send(PG_FUNCTION_ARGS) * Optionally determines the collation. */ static int -mcv_match_expression(Node *expr, Bitmapset *keys, List *exprs, Oid *collid) +mcv_match_expression(Node *expr, List *exprs, Oid *collid) { - int idx; + bool expr_is_var = IsA(expr, Var); + int idx = 0; + ListCell *lc; - if (IsA(expr, Var)) - { - /* simple Var, so just lookup using varattno */ - Var *var = (Var *) expr; - - if (collid) - *collid = var->varcollid; - - idx = bms_member_index(keys, var->varattno); + if (collid) + *collid = expr_is_var ? ((Var *) expr)->varcollid : exprCollation(expr); - if (idx < 0) - elog(ERROR, "variable not found in statistics object"); - } - else + foreach(lc, exprs) { - /* expression - lookup in stats expressions */ - ListCell *lc; + Node *node = (Node *) lfirst(lc); - if (collid) - *collid = exprCollation(expr); - - /* expressions are stored after the simple columns */ - idx = bms_num_members(keys); - foreach(lc, exprs) + if (expr_is_var) { - Node *stat_expr = (Node *) lfirst(lc); - - if (equal(expr, stat_expr)) - break; - - idx++; + if (IsA(node, Var) && + ((Var *) node)->varattno == ((Var *) expr)->varattno) + return idx; } + else if (equal(expr, node)) + return idx; - if (lc == NULL) - elog(ERROR, "expression not found in statistics object"); + idx++; } - return idx; + if (expr_is_var) + elog(ERROR, "variable not found in statistics object"); + else + elog(ERROR, "expression not found in statistics object"); + + return -1; /* keep compiler quiet */ } /* @@ -1594,7 +1583,7 @@ mcv_match_expression(Node *expr, Bitmapset *keys, List *exprs, Oid *collid) */ static bool * mcv_get_match_bitmap(PlannerInfo *root, List *clauses, - Bitmapset *keys, List *exprs, + List *exprs, MCVList *mcvlist, bool is_or) { ListCell *l; @@ -1644,7 +1633,7 @@ mcv_get_match_bitmap(PlannerInfo *root, List *clauses, elog(ERROR, "incompatible clause"); /* match the attribute/expression to a dimension of the statistic */ - idx = mcv_match_expression(clause_expr, keys, exprs, &collid); + idx = mcv_match_expression(clause_expr, exprs, &collid); /* * Walk through the MCV items and evaluate the current clause. We @@ -1751,7 +1740,7 @@ mcv_get_match_bitmap(PlannerInfo *root, List *clauses, } /* match the attribute/expression to a dimension of the statistic */ - idx = mcv_match_expression(clause_expr, keys, exprs, &collid); + idx = mcv_match_expression(clause_expr, exprs, &collid); /* * Walk through the MCV items and evaluate the current clause. We @@ -1822,7 +1811,7 @@ mcv_get_match_bitmap(PlannerInfo *root, List *clauses, Node *clause_expr = (Node *) (expr->arg); /* match the attribute/expression to a dimension of the statistic */ - int idx = mcv_match_expression(clause_expr, keys, exprs, NULL); + int idx = mcv_match_expression(clause_expr, exprs, NULL); /* * Walk through the MCV items and evaluate the current clause. We @@ -1864,7 +1853,7 @@ mcv_get_match_bitmap(PlannerInfo *root, List *clauses, Assert(list_length(bool_clauses) >= 2); /* build the match bitmap for the OR-clauses */ - bool_matches = mcv_get_match_bitmap(root, bool_clauses, keys, exprs, + bool_matches = mcv_get_match_bitmap(root, bool_clauses, exprs, mcvlist, is_orclause(clause)); /* @@ -1891,7 +1880,7 @@ mcv_get_match_bitmap(PlannerInfo *root, List *clauses, Assert(list_length(not_args) == 1); /* build the match bitmap for the NOT-clause */ - not_matches = mcv_get_match_bitmap(root, not_args, keys, exprs, + not_matches = mcv_get_match_bitmap(root, not_args, exprs, mcvlist, false); /* @@ -1911,7 +1900,7 @@ mcv_get_match_bitmap(PlannerInfo *root, List *clauses, Var *var = (Var *) (clause); /* match the attribute to a dimension of the statistic */ - int idx = bms_member_index(keys, var->varattno); + int idx = mcv_match_expression((Node *) var, exprs, NULL); Assert(var->vartype == BOOLOID); @@ -1939,7 +1928,7 @@ mcv_get_match_bitmap(PlannerInfo *root, List *clauses, int idx; /* match the expression to a dimension of the statistic */ - idx = mcv_match_expression(clause, keys, exprs, NULL); + idx = mcv_match_expression(clause, exprs, NULL); /* * Walk through the MCV items and evaluate the current clause. We @@ -2057,7 +2046,7 @@ mcv_clauselist_selectivity(PlannerInfo *root, StatisticExtInfo *stat, mcv = statext_mcv_load(stat->statOid, rte->inh); /* build a match bitmap for the clauses */ - matches = mcv_get_match_bitmap(root, clauses, stat->keys, stat->exprs, + matches = mcv_get_match_bitmap(root, clauses, stat->exprs, mcv, false); /* sum frequencies for all the matching MCV items */ @@ -2130,8 +2119,8 @@ mcv_clause_selectivity_or(PlannerInfo *root, StatisticExtInfo *stat, *or_matches = palloc0_array(bool, mcv->nitems); /* build the match bitmap for the new clause */ - new_matches = mcv_get_match_bitmap(root, list_make1(clause), stat->keys, - stat->exprs, mcv, false); + new_matches = mcv_get_match_bitmap(root, list_make1(clause), stat->exprs, + mcv, false); /* * Sum the frequencies for all the MCV items matching this clause and also diff --git a/src/backend/statistics/mvdistinct.c b/src/backend/statistics/mvdistinct.c index 0add6455f6a..121196e1d6c 100644 --- a/src/backend/statistics/mvdistinct.c +++ b/src/backend/statistics/mvdistinct.c @@ -128,6 +128,12 @@ statext_ndistinct_build(double totalrows, StatsBuildData *data) Assert(AttributeNumberIsValid(item->attributes[j])); } + /* + * Order the attnums the way dump and restore needs (see + * compare_attnums). + */ + qsort(item->attributes, k, sizeof(AttrNumber), compare_attnums); + /* release memory used by ndistinct calculation */ oldcxt = MemoryContextSwitchTo(cxt); item->ndistinct = @@ -354,13 +360,13 @@ statext_ndistinct_free(MVNDistinct *ndistinct) * attributes list correspond to attnums/expressions defined by the extended * statistics object. * - * Positive attnums are attributes which must be found in the stxkeys, - * while negative attnums correspond to an expression number, no attribute + * Positive attnums correspond to table columns (excluding virtual generated + * columns), while negative attnums correspond to expressions. No attribute * number can be below (0 - numexprs). */ bool statext_ndistinct_validate(const MVNDistinct *ndistinct, - const int2vector *stxkeys, + const Bitmapset *keys, int numexprs, int elevel) { int attnum_expr_lowbound = 0 - numexprs; @@ -381,15 +387,8 @@ statext_ndistinct_validate(const MVNDistinct *ndistinct, if (attnum > 0) { - /* attribute number in stxkeys */ - for (int k = 0; k < stxkeys->dim1; k++) - { - if (attnum == stxkeys->values[k]) - { - ok = true; - break; - } - } + /* attribute number in keys */ + ok = bms_is_member(attnum, keys); } else if ((attnum < 0) && (attnum >= attnum_expr_lowbound)) { diff --git a/src/backend/utils/adt/ruleutils.c b/src/backend/utils/adt/ruleutils.c index e9c1c9db1b1..58a828ae319 100644 --- a/src/backend/utils/adt/ruleutils.c +++ b/src/backend/utils/adt/ruleutils.c @@ -57,6 +57,7 @@ #include "rewrite/rewriteHandler.h" #include "rewrite/rewriteManip.h" #include "rewrite/rewriteSupport.h" +#include "statistics/statistics.h" #include "utils/array.h" #include "utils/builtins.h" #include "utils/fmgroids.h" @@ -361,7 +362,7 @@ static char *pg_get_indexdef_worker(Oid indexrelid, int colno, bool attrsOnly, bool keysOnly, bool showTblSpc, bool inherits, int prettyFlags, bool missing_ok); -static char *pg_get_statisticsobj_worker(Oid statextid, bool columns_only, +static char *pg_get_statisticsobj_worker(Oid statextid, bool missing_ok); static char *pg_get_partkeydef_worker(Oid relid, int prettyFlags, bool attrsOnly, bool missing_ok); @@ -1613,7 +1614,7 @@ pg_get_statisticsobjdef(PG_FUNCTION_ARGS) Oid statextid = PG_GETARG_OID(0); char *res; - res = pg_get_statisticsobj_worker(statextid, false, true); + res = pg_get_statisticsobj_worker(statextid, true); if (res == NULL) PG_RETURN_NULL(); @@ -1628,7 +1629,27 @@ pg_get_statisticsobjdef(PG_FUNCTION_ARGS) char * pg_get_statisticsobjdef_string(Oid statextid) { - return pg_get_statisticsobj_worker(statextid, false, false); + return pg_get_statisticsobj_worker(statextid, false); +} + +/* + * Deparse one column or expression of an extended statistics object into the + * text used in its CREATE STATISTICS command. + */ +static char * +deparse_stat_entry(Node *expr, Oid relid, List *context) +{ + char *str; + + if (IsA(expr, Var) && ((Var *) expr)->varattno > 0) + return pstrdup(quote_identifier( + get_attname(relid, ((Var *) expr)->varattno, false))); + + str = deparse_expression_pretty(expr, context, false, false, + PRETTYFLAG_PAREN, 0); + if (looks_like_function(expr)) + return str; + return psprintf("(%s)", str); } /* @@ -1639,21 +1660,57 @@ Datum pg_get_statisticsobjdef_columns(PG_FUNCTION_ARGS) { Oid statextid = PG_GETARG_OID(0); - char *res; + Form_pg_statistic_ext statextrec; + HeapTuple statexttup; + Datum datum; + List *allexprs = NIL; + char *tmp; + List *context; + ListCell *lc; + ArrayBuildState *astate = NULL; - res = pg_get_statisticsobj_worker(statextid, true, true); + statexttup = SearchSysCache1(STATEXTOID, ObjectIdGetDatum(statextid)); - if (res == NULL) + if (!HeapTupleIsValid(statexttup)) PG_RETURN_NULL(); - PG_RETURN_TEXT_P(string_to_text(res)); + statextrec = (Form_pg_statistic_ext) GETSTRUCT(statexttup); + + datum = SysCacheGetAttrNotNull(STATEXTOID, statexttup, + Anum_pg_statistic_ext_stxexprs); + tmp = TextDatumGetCString(datum); + allexprs = (List *) stringToNode(tmp); + pfree(tmp); + + context = deparse_context_for(get_relation_name(statextrec->stxrelid), + statextrec->stxrelid); + + foreach(lc, allexprs) + { + Node *expr = (Node *) lfirst(lc); + char *str; + + str = deparse_stat_entry(expr, statextrec->stxrelid, context); + astate = accumArrayResult(astate, + PointerGetDatum(cstring_to_text(str)), + false, + TEXTOID, + CurrentMemoryContext); + } + + ReleaseSysCache(statexttup); + + if (astate == NULL) + PG_RETURN_NULL(); + + PG_RETURN_DATUM(makeArrayResult(astate, CurrentMemoryContext)); } /* * Internal workhorse to decompile an extended statistics object. */ static char * -pg_get_statisticsobj_worker(Oid statextid, bool columns_only, bool missing_ok) +pg_get_statisticsobj_worker(Oid statextid, bool missing_ok) { Form_pg_statistic_ext statextrec; HeapTuple statexttup; @@ -1663,6 +1720,7 @@ pg_get_statisticsobj_worker(Oid statextid, bool columns_only, bool missing_ok) ArrayType *arr; char *enabled; Datum datum; + char *exprsString; bool ndistinct_enabled; bool dependencies_enabled; bool mcv_enabled; @@ -1670,7 +1728,6 @@ pg_get_statisticsobj_worker(Oid statextid, bool columns_only, bool missing_ok) List *context; ListCell *lc; List *exprs = NIL; - bool has_exprs; int ncolumns; statexttup = SearchSysCache1(STATEXTOID, ObjectIdGetDatum(statextid)); @@ -1682,151 +1739,113 @@ pg_get_statisticsobj_worker(Oid statextid, bool columns_only, bool missing_ok) elog(ERROR, "cache lookup failed for statistics object %u", statextid); } - /* has the statistics expressions? */ - has_exprs = !heap_attisnull(statexttup, Anum_pg_statistic_ext_stxexprs, NULL); - statextrec = (Form_pg_statistic_ext) GETSTRUCT(statexttup); /* - * Get the statistics expressions, if any. (NOTE: we do not use the - * relcache versions of the expressions, because we want to display + * Get the statistics expressions. (NOTE: we do not use the relcache + * versions of the expressions, because we want to display * non-const-folded expressions.) */ - if (has_exprs) - { - Datum exprsDatum; - char *exprsString; - - exprsDatum = SysCacheGetAttrNotNull(STATEXTOID, statexttup, - Anum_pg_statistic_ext_stxexprs); - exprsString = TextDatumGetCString(exprsDatum); - exprs = (List *) stringToNode(exprsString); - pfree(exprsString); - } - else - exprs = NIL; + datum = SysCacheGetAttrNotNull(STATEXTOID, statexttup, + Anum_pg_statistic_ext_stxexprs); + exprsString = TextDatumGetCString(datum); + exprs = (List *) stringToNode(exprsString); + pfree(exprsString); /* count the number of columns (attributes and expressions) */ - ncolumns = statextrec->stxkeys.dim1 + list_length(exprs); + ncolumns = list_length(exprs); initStringInfo(&buf); - if (!columns_only) - { - nsp = get_namespace_name_or_temp(statextrec->stxnamespace); - appendStringInfo(&buf, "CREATE STATISTICS %s", - quote_qualified_identifier(nsp, - NameStr(statextrec->stxname))); - - /* - * Decode the stxkind column so that we know which stats types to - * print. - */ - datum = SysCacheGetAttrNotNull(STATEXTOID, statexttup, - Anum_pg_statistic_ext_stxkind); - arr = DatumGetArrayTypeP(datum); - if (ARR_NDIM(arr) != 1 || - ARR_HASNULL(arr) || - ARR_ELEMTYPE(arr) != CHAROID) - elog(ERROR, "stxkind is not a 1-D char array"); - enabled = (char *) ARR_DATA_PTR(arr); + nsp = get_namespace_name_or_temp(statextrec->stxnamespace); + appendStringInfo(&buf, "CREATE STATISTICS %s", + quote_qualified_identifier(nsp, + NameStr(statextrec->stxname))); - ndistinct_enabled = false; - dependencies_enabled = false; - mcv_enabled = false; - - for (i = 0; i < ARR_DIMS(arr)[0]; i++) - { - if (enabled[i] == STATS_EXT_NDISTINCT) - ndistinct_enabled = true; - else if (enabled[i] == STATS_EXT_DEPENDENCIES) - dependencies_enabled = true; - else if (enabled[i] == STATS_EXT_MCV) - mcv_enabled = true; - - /* ignore STATS_EXT_EXPRESSIONS (it's built automatically) */ - } + /* + * Decode the stxkind column so that we know which stats types to print. + */ + datum = SysCacheGetAttrNotNull(STATEXTOID, statexttup, + Anum_pg_statistic_ext_stxkind); + arr = DatumGetArrayTypeP(datum); + if (ARR_NDIM(arr) != 1 || + ARR_HASNULL(arr) || + ARR_ELEMTYPE(arr) != CHAROID) + elog(ERROR, "stxkind is not a 1-D char array"); + enabled = (char *) ARR_DATA_PTR(arr); - /* - * If any option is disabled, then we'll need to append the types - * clause to show which options are enabled. We omit the types clause - * on purpose when all options are enabled, so a pg_dump/pg_restore - * will create all statistics types on a newer postgres version, if - * the statistics had all options enabled on the original version. - * - * But if the statistics is defined on just a single column, it has to - * be an expression statistics. In that case we don't need to specify - * kinds. - */ - if ((!ndistinct_enabled || !dependencies_enabled || !mcv_enabled) && - (ncolumns > 1)) - { - bool gotone = false; + ndistinct_enabled = false; + dependencies_enabled = false; + mcv_enabled = false; - appendStringInfoString(&buf, " ("); + for (i = 0; i < ARR_DIMS(arr)[0]; i++) + { + if (enabled[i] == STATS_EXT_NDISTINCT) + ndistinct_enabled = true; + else if (enabled[i] == STATS_EXT_DEPENDENCIES) + dependencies_enabled = true; + else if (enabled[i] == STATS_EXT_MCV) + mcv_enabled = true; - if (ndistinct_enabled) - { - appendStringInfoString(&buf, "ndistinct"); - gotone = true; - } + /* ignore STATS_EXT_EXPRESSIONS (it's built automatically) */ + } - if (dependencies_enabled) - { - appendStringInfo(&buf, "%sdependencies", gotone ? ", " : ""); - gotone = true; - } + /* + * If any option is disabled, then we'll need to append the types clause + * to show which options are enabled. We omit the types clause on purpose + * when all options are enabled, so a pg_dump/pg_restore will create all + * statistics types on a newer postgres version, if the statistics had all + * options enabled on the original version. + * + * But if the statistics is defined on just a single column, it has to be + * an expression statistics. In that case we don't need to specify kinds. + */ + if ((!ndistinct_enabled || !dependencies_enabled || !mcv_enabled) && + (ncolumns > 1)) + { + bool gotone = false; - if (mcv_enabled) - appendStringInfo(&buf, "%smcv", gotone ? ", " : ""); + appendStringInfoString(&buf, " ("); - appendStringInfoChar(&buf, ')'); + if (ndistinct_enabled) + { + appendStringInfoString(&buf, "ndistinct"); + gotone = true; } - appendStringInfoString(&buf, " ON "); - } - - /* decode simple column references */ - for (colno = 0; colno < statextrec->stxkeys.dim1; colno++) - { - AttrNumber attnum = statextrec->stxkeys.values[colno]; - char *attname; - - if (colno > 0) - appendStringInfoString(&buf, ", "); + if (dependencies_enabled) + { + appendStringInfo(&buf, "%sdependencies", gotone ? ", " : ""); + gotone = true; + } - attname = get_attname(statextrec->stxrelid, attnum, false); + if (mcv_enabled) + appendStringInfo(&buf, "%smcv", gotone ? ", " : ""); - appendStringInfoString(&buf, quote_identifier(attname)); + appendStringInfoChar(&buf, ')'); } + appendStringInfoString(&buf, " ON "); + context = deparse_context_for(get_relation_name(statextrec->stxrelid), statextrec->stxrelid); + colno = 0; foreach(lc, exprs) { Node *expr = (Node *) lfirst(lc); - char *str; - int prettyFlags = PRETTYFLAG_PAREN; - - str = deparse_expression_pretty(expr, context, false, false, - prettyFlags, 0); if (colno > 0) appendStringInfoString(&buf, ", "); - /* Need parens if it's not a bare function call */ - if (looks_like_function(expr)) - appendStringInfoString(&buf, str); - else - appendStringInfo(&buf, "(%s)", str); - + appendStringInfoString(&buf, + deparse_stat_entry(expr, statextrec->stxrelid, + context)); colno++; } - if (!columns_only) - appendStringInfo(&buf, " FROM %s", - generate_relation_name(statextrec->stxrelid, NIL)); + appendStringInfo(&buf, " FROM %s", + generate_relation_name(statextrec->stxrelid, NIL)); ReleaseSysCache(statexttup); @@ -1843,10 +1862,11 @@ pg_get_statisticsobjdef_expressions(PG_FUNCTION_ARGS) Form_pg_statistic_ext statextrec; HeapTuple statexttup; Datum datum; + Relation rel; List *context; ListCell *lc; - List *exprs = NIL; - bool has_exprs; + List *rawexprs = NIL; + Bitmapset *columns; char *tmp; ArrayBuildState *astate = NULL; @@ -1855,37 +1875,41 @@ pg_get_statisticsobjdef_expressions(PG_FUNCTION_ARGS) if (!HeapTupleIsValid(statexttup)) PG_RETURN_NULL(); - /* Does the stats object have expressions? */ - has_exprs = !heap_attisnull(statexttup, Anum_pg_statistic_ext_stxexprs, NULL); - - /* no expressions? we're done */ - if (!has_exprs) - { - ReleaseSysCache(statexttup); - PG_RETURN_NULL(); - } - statextrec = (Form_pg_statistic_ext) GETSTRUCT(statexttup); /* - * Get the statistics expressions, and deparse them into text values. + * Decode stxexprs as originally written; we deparse these, because we + * want to display the expressions without const-folding. */ datum = SysCacheGetAttrNotNull(STATEXTOID, statexttup, Anum_pg_statistic_ext_stxexprs); tmp = TextDatumGetCString(datum); - exprs = (List *) stringToNode(tmp); + rawexprs = (List *) stringToNode(tmp); pfree(tmp); + /* + * Skip the plain columns: their statistics are in pg_statistic, not in + * pg_statistic_ext_data.stxdexpr. What remains are the expressions, in + * the same order as stxdexpr, which pg_stats_ext_exprs pairs this output + * with. + */ + rel = table_open(statextrec->stxrelid, AccessShareLock); + columns = statext_column_entries(rawexprs, rel); + table_close(rel, AccessShareLock); + context = deparse_context_for(get_relation_name(statextrec->stxrelid), statextrec->stxrelid); - foreach(lc, exprs) + foreach(lc, rawexprs) { - Node *expr = (Node *) lfirst(lc); + Node *rawexpr = (Node *) lfirst(lc); char *str; int prettyFlags = PRETTYFLAG_INDENT; - str = deparse_expression_pretty(expr, context, false, false, + if (bms_is_member(foreach_current_index(lc), columns)) + continue; + + str = deparse_expression_pretty(rawexpr, context, false, false, prettyFlags, 0); astate = accumArrayResult(astate, @@ -1897,6 +1921,9 @@ pg_get_statisticsobjdef_expressions(PG_FUNCTION_ARGS) ReleaseSysCache(statexttup); + if (astate == NULL) + PG_RETURN_NULL(); + PG_RETURN_DATUM(makeArrayResult(astate, CurrentMemoryContext)); } diff --git a/src/backend/utils/adt/selfuncs.c b/src/backend/utils/adt/selfuncs.c index 5ee377595bd..fbc37d9292c 100644 --- a/src/backend/utils/adt/selfuncs.c +++ b/src/backend/utils/adt/selfuncs.c @@ -4626,7 +4626,7 @@ estimate_multivariate_ndistinct(PlannerInfo *root, RelOptInfo *rel, if (!AttrNumberIsForUserDefinedAttr(attnum)) continue; - if (bms_is_member(attnum, info->keys)) + if (stat_covers_attnum(info, attnum)) nshared_vars++; continue; @@ -4691,14 +4691,15 @@ estimate_multivariate_ndistinct(PlannerInfo *root, RelOptInfo *rel, ListCell *lc2; Bitmapset *matched = NULL; AttrNumber attnum_offset; + int nexprs = stat_num_expressions(matched_info); /* * How much we need to offset the attnums? If there are no * expressions, no offset is needed. Otherwise offset enough to move * the lowest one (which is equal to number of expressions) to 1. */ - if (matched_info->exprs) - attnum_offset = (list_length(matched_info->exprs) + 1); + if (nexprs > 0) + attnum_offset = nexprs + 1; else attnum_offset = 0; @@ -4728,7 +4729,7 @@ estimate_multivariate_ndistinct(PlannerInfo *root, RelOptInfo *rel, continue; /* Is the variable covered by the statistics object? */ - if (!bms_is_member(attnum, matched_info->keys)) + if (!stat_covers_attnum(matched_info, attnum)) continue; attnum = attnum + attnum_offset; @@ -4755,6 +4756,13 @@ estimate_multivariate_ndistinct(PlannerInfo *root, RelOptInfo *rel, { Node *expr = (Node *) lfirst(lc3); + /* + * columns have no per-expression stats, so count expressions + * only + */ + if (bms_is_member(foreach_current_index(lc3), matched_info->columns)) + continue; + if (equal(varinfo->var, expr)) { AttrNumber attnum = -(idx + 1); @@ -5921,6 +5929,13 @@ examine_variable(PlannerInfo *root, Node *node, int varRelid, { Node *expr = (Node *) lfirst(expr_item); + /* + * columns have no per-expression stats, so count expressions + * only + */ + if (bms_is_member(foreach_current_index(expr_item), info->columns)) + continue; + Assert(expr); /* strip RelabelType before comparing it */ diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c index e4a4ba47280..5f9f00974df 100644 --- a/src/bin/psql/describe.c +++ b/src/bin/psql/describe.c @@ -2675,8 +2675,14 @@ describeOneTableDetails(const char *schemaname, "SELECT oid, " "stxrelid::pg_catalog.regclass, " "stxnamespace::pg_catalog.regnamespace::pg_catalog.text AS nsp, " - "stxname,\n" - "pg_catalog.pg_get_statisticsobjdef_columns(oid) AS columns,\n" + "stxname,\n"); + if (pset.sversion >= 200000) + appendPQExpBufferStr(&buf, + "pg_catalog.array_to_string(pg_catalog.pg_get_statisticsobjdef_columns(oid), ', ') AS columns,\n"); + else + appendPQExpBufferStr(&buf, + "pg_catalog.pg_get_statisticsobjdef_columns(oid) AS columns,\n"); + appendPQExpBuffer(&buf, " " CppAsString2(STATS_EXT_NDISTINCT) " = any(stxkind) AS ndist_enabled,\n" " " CppAsString2(STATS_EXT_DEPENDENCIES) " = any(stxkind) AS deps_enabled,\n" " " CppAsString2(STATS_EXT_MCV) " = any(stxkind) AS mcv_enabled,\n" @@ -4831,7 +4837,13 @@ listExtendedStats(const char *pattern, bool verbose) gettext_noop("Schema"), gettext_noop("Name")); - if (pset.sversion >= 140000) + if (pset.sversion >= 200000) + appendPQExpBuffer(&buf, + "pg_catalog.format('%%s FROM %%s', \n" + " pg_catalog.array_to_string(pg_catalog.pg_get_statisticsobjdef_columns(es.oid), ', '), \n" + " es.stxrelid::pg_catalog.regclass) AS \"%s\"", + gettext_noop("Definition")); + else if (pset.sversion >= 140000) appendPQExpBuffer(&buf, "pg_catalog.format('%%s FROM %%s', \n" " pg_catalog.pg_get_statisticsobjdef_columns(es.oid), \n" diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 50fb182cb79..741e1d45f90 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -3995,9 +3995,9 @@ proname => 'pg_get_statisticsobjdef', provolatile => 's', prorettype => 'text', proargtypes => 'oid', prosrc => 'pg_get_statisticsobjdef' }, -{ oid => '6174', descr => 'extended statistics columns', +{ oid => '6174', descr => 'extended statistics columns and expressions', proname => 'pg_get_statisticsobjdef_columns', provolatile => 's', - prorettype => 'text', proargtypes => 'oid', + prorettype => '_text', proargtypes => 'oid', prosrc => 'pg_get_statisticsobjdef_columns' }, { oid => '6173', descr => 'extended statistics expressions', proname => 'pg_get_statisticsobjdef_expressions', provolatile => 's', diff --git a/src/include/catalog/pg_statistic_ext.h b/src/include/catalog/pg_statistic_ext.h index bf8b495e8f8..08ed6e23287 100644 --- a/src/include/catalog/pg_statistic_ext.h +++ b/src/include/catalog/pg_statistic_ext.h @@ -46,19 +46,15 @@ CATALOG(pg_statistic_ext,3381,StatisticExtRelationId) Oid stxowner BKI_LOOKUP(pg_authid); /* statistics object's owner */ - /* - * variable-length/nullable fields start here, but we allow direct access - * to stxkeys - */ - int2vector stxkeys BKI_FORCE_NOT_NULL; /* array of column keys */ - #ifdef CATALOG_VARLEN int16 stxstattarget BKI_DEFAULT(_null_) BKI_FORCE_NULL; /* statistics target */ char stxkind[] BKI_FORCE_NOT_NULL; /* statistics kinds requested * to build */ - pg_node_tree stxexprs; /* A list of expression trees for stats - * attributes that are not simple column - * references. */ + pg_node_tree stxexprs BKI_FORCE_NOT_NULL; /* expression trees for the + * columns and expressions the + * statistics object is + * defined on; simple columns + * are Var nodes */ #endif } FormData_pg_statistic_ext; @@ -81,7 +77,6 @@ DECLARE_INDEX(pg_statistic_ext_relid_index, 3379, StatisticExtRelidIndexId, pg_s MAKE_SYSCACHE(STATEXTOID, pg_statistic_ext_oid_index, 4); MAKE_SYSCACHE(STATEXTNAMENSP, pg_statistic_ext_name_index, 4); -DECLARE_ARRAY_FOREIGN_KEY((stxrelid, stxkeys), pg_attribute, (attrelid, attnum)); #ifdef EXPOSE_TO_CLIENT_CODE diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index 2ba1dbfb67f..2f63d7b2b59 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -1528,11 +1528,11 @@ typedef struct StatisticExtInfo /* statistics kind of this entry */ char kind; - /* attnums of the columns covered */ - Bitmapset *keys; - - /* expressions */ + /* all columns and expressions the object is defined on, in declared order */ List *exprs; + + /* positions in "exprs" of the entries that are plain columns */ + Bitmapset *columns; } StatisticExtInfo; /* diff --git a/src/include/statistics/extended_stats_internal.h b/src/include/statistics/extended_stats_internal.h index 4d7151ab11d..844b149a7cb 100644 --- a/src/include/statistics/extended_stats_internal.h +++ b/src/include/statistics/extended_stats_internal.h @@ -73,7 +73,7 @@ extern MVNDistinct *statext_ndistinct_build(double totalrows, StatsBuildData *da extern bytea *statext_ndistinct_serialize(MVNDistinct *ndistinct); extern MVNDistinct *statext_ndistinct_deserialize(bytea *data); extern bool statext_ndistinct_validate(const MVNDistinct *ndistinct, - const int2vector *stxkeys, + const Bitmapset *keys, int numexprs, int elevel); extern void statext_ndistinct_free(MVNDistinct *ndistinct); @@ -81,7 +81,7 @@ extern MVDependencies *statext_dependencies_build(StatsBuildData *data); extern bytea *statext_dependencies_serialize(MVDependencies *dependencies); extern MVDependencies *statext_dependencies_deserialize(bytea *data); extern bool statext_dependencies_validate(const MVDependencies *dependencies, - const int2vector *stxkeys, + const Bitmapset *keys, int numexprs, int elevel); extern void statext_dependencies_free(MVDependencies *dependencies); @@ -106,8 +106,7 @@ extern int multi_sort_compare_dims(int start, int end, const SortItem *a, const SortItem *b, MultiSortSupport mss); extern int compare_scalars_simple(const void *a, const void *b, void *arg); extern int compare_datums_simple(Datum a, Datum b, SortSupport ssup); - -extern AttrNumber *build_attnums_array(Bitmapset *attrs, int nexprs, int *numattrs); +extern int compare_attnums(const void *a, const void *b); extern SortItem *build_sorted_items(StatsBuildData *data, int *nitems, MultiSortSupport mss, diff --git a/src/include/statistics/statistics.h b/src/include/statistics/statistics.h index ff0cea4b0c5..888d3e27cc7 100644 --- a/src/include/statistics/statistics.h +++ b/src/include/statistics/statistics.h @@ -127,6 +127,12 @@ extern StatisticExtInfo *choose_best_statistics(List *stats, char requiredkind, List **clause_exprs, int nclauses); extern HeapTuple statext_expressions_load(Oid stxoid, bool inh, int idx); +extern List *statext_get_stxexprs(HeapTuple htup, Relation rel, + Bitmapset **columns); +extern Bitmapset *statext_column_entries(List *exprs, Relation rel); +extern bool stat_covers_attnum(StatisticExtInfo *stat, AttrNumber attnum); +extern int stat_num_expressions(StatisticExtInfo *stat); +extern Node *stat_nth_expression(StatisticExtInfo *stat, int n); extern bool import_relation_statistics(Relation rel, const NullableDatum *version, diff --git a/src/test/regress/expected/create_table_like.out b/src/test/regress/expected/create_table_like.out index 0deba8b6ac4..e8414bfe0fe 100644 --- a/src/test/regress/expected/create_table_like.out +++ b/src/test/regress/expected/create_table_like.out @@ -712,7 +712,7 @@ SELECT attname, attcompression FROM pg_attribute (5 rows) -- LIKE ... INCLUDING STATISTICS with dropped columns in the parent, --- so stxkeys attnums are not contiguous. +-- so column attnums are not contiguous. CREATE TABLE ctl_stats3_parent (a int, b int, c int); ALTER TABLE ctl_stats3_parent DROP COLUMN b; CREATE STATISTICS ctl_stats3_stat ON a, c FROM ctl_stats3_parent; @@ -722,15 +722,10 @@ ALTER TABLE ctl_stats4_parent DROP COLUMN b; CREATE STATISTICS ctl_stats4_stat ON a, c FROM ctl_stats4_parent; CREATE TABLE ctl_stats4_child (LIKE ctl_stats4_parent INCLUDING STATISTICS); SELECT s.stxrelid::regclass AS relation, - array_agg(a.attname ORDER BY u.ord) AS stats_columns + pg_get_statisticsobjdef_columns(s.oid) AS stats_columns FROM pg_statistic_ext s -CROSS JOIN LATERAL - unnest(s.stxkeys::int2[]) WITH ORDINALITY AS u(attnum, ord) -JOIN pg_attribute a - ON a.attrelid = s.stxrelid AND a.attnum = u.attnum WHERE s.stxrelid IN ('ctl_stats3_child'::regclass, 'ctl_stats4_child'::regclass) -GROUP BY s.stxrelid ORDER BY s.stxrelid::regclass::text; relation | stats_columns ------------------+--------------- diff --git a/src/test/regress/expected/oidjoins.out b/src/test/regress/expected/oidjoins.out index 51b9608a668..419da49663a 100644 --- a/src/test/regress/expected/oidjoins.out +++ b/src/test/regress/expected/oidjoins.out @@ -166,7 +166,6 @@ NOTICE: checking pg_statistic {starelid,staattnum} => pg_attribute {attrelid,at NOTICE: checking pg_statistic_ext {stxrelid} => pg_class {oid} NOTICE: checking pg_statistic_ext {stxnamespace} => pg_namespace {oid} NOTICE: checking pg_statistic_ext {stxowner} => pg_authid {oid} -NOTICE: checking pg_statistic_ext {stxrelid,stxkeys} => pg_attribute {attrelid,attnum} NOTICE: checking pg_statistic_ext_data {stxoid} => pg_statistic_ext {oid} NOTICE: checking pg_rewrite {ev_class} => pg_class {oid} NOTICE: checking pg_trigger {tgrelid} => pg_class {oid} diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index acbe4667d89..9a144be96a4 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -2722,10 +2722,7 @@ pg_stats_ext| SELECT cn.nspname AS schemaname, s.stxname AS statistics_name, s.oid AS statistics_id, pg_get_userbyid(s.stxowner) AS statistics_owner, - ( SELECT array_agg(a.attname ORDER BY a.attnum) AS array_agg - FROM (unnest(s.stxkeys) k(k) - JOIN pg_attribute a ON (((a.attrelid = s.stxrelid) AND (a.attnum = k.k))))) AS attnames, - pg_get_statisticsobjdef_expressions(s.oid) AS exprs, + pg_get_statisticsobjdef_columns(s.oid) AS exprs, s.stxkind AS kinds, sd.stxdinherit AS inherited, sd.stxdndistinct AS n_distinct, diff --git a/src/test/regress/expected/stats_ext.out b/src/test/regress/expected/stats_ext.out index 42d6c2777ea..a8daa480b19 100644 --- a/src/test/regress/expected/stats_ext.out +++ b/src/test/regress/expected/stats_ext.out @@ -332,6 +332,95 @@ SELECT * FROM check_estimated_rows('SELECT a + 1, b FROM ONLY stxdinp GROUP BY 1 (1 row) DROP TABLE stxdinp; +-- a column-equivalent expression (one that const-folds to a plain column, or a +-- passthrough virtual generated column) is still an expression, not that column +CREATE TABLE column_equivalent (a int, b int, g int GENERATED ALWAYS AS (a) VIRTUAL) WITH (autovacuum_enabled = off); +INSERT INTO column_equivalent (a, b) SELECT i%100, i%10 FROM generate_series(1, 10000) i; +-- a column-equivalent CASE between two complex expressions +CREATE STATISTICS column_equivalent_middle ON (a + b), (CASE WHEN true THEN a ELSE b END), (a * b) FROM column_equivalent; +-- only column-equivalent entries, which all fold to plain columns +CREATE STATISTICS column_equivalent_all ON (CASE WHEN true THEN a END), b FROM column_equivalent; +ANALYZE column_equivalent; +-- a column-equivalent CASE is reported as an expression, with its own +-- statistics, like the other expressions +SELECT statistics_name, expr, n_distinct +FROM pg_stats_ext_exprs +WHERE statistics_name IN ('column_equivalent_middle', 'column_equivalent_all') +ORDER BY statistics_name, expr; + statistics_name | expr | n_distinct +--------------------------+------------------------+------------ + column_equivalent_all | +| 100 + | CASE +| + | WHEN true THEN a +| + | ELSE NULL::integer+| + | END | + column_equivalent_middle | +| 100 + | CASE +| + | WHEN true THEN a +| + | ELSE b +| + | END | + column_equivalent_middle | (a * b) | 84 + column_equivalent_middle | (a + b) | 55 +(4 rows) + +-- each complex expression's GROUP BY estimate uses its own statistics +SELECT * FROM check_estimated_rows('SELECT (a + b) FROM column_equivalent GROUP BY 1'); + estimated | actual +-----------+-------- + 55 | 55 +(1 row) + +SELECT * FROM check_estimated_rows('SELECT (a * b) FROM column_equivalent GROUP BY 1'); + estimated | actual +-----------+-------- + 84 | 84 +(1 row) + +-- column_equivalent_all is not used for a GROUP BY on the plain columns a, b, +-- as its first entry is an expression rather than column a +SELECT * FROM check_estimated_rows('SELECT COUNT(*) FROM column_equivalent GROUP BY a, b'); + estimated | actual +-----------+-------- + 1000 | 100 +(1 row) + +-- a parenthesized column is the same column, so it is rejected as a duplicate +CREATE STATISTICS column_equivalent_dup_paren ON a, (a) FROM column_equivalent; +ERROR: duplicate column name in statistics definition +-- but an expression that const-folds to a column already listed is not a +-- duplicate, +CREATE STATISTICS column_equivalent_with_case ON a, (CASE WHEN true THEN a END), b FROM column_equivalent; +-- and neither is a passthrough generated column +CREATE STATISTICS column_equivalent_with_vgen ON a, g FROM column_equivalent; +-- a generated column listed twice is a duplicate expression +CREATE STATISTICS column_equivalent_dup_vgen ON g, g FROM column_equivalent; +ERROR: duplicate expression in statistics definition +-- a single entry that const-folds to a plain column, and a single passthrough +-- generated column; each is an expression, with its own statistics +CREATE STATISTICS column_equivalent_single_case ON (CASE WHEN true THEN a END) FROM column_equivalent; +CREATE STATISTICS column_equivalent_single_vgen ON g FROM column_equivalent; +SELECT stxname, stxkind FROM pg_statistic_ext +WHERE stxname LIKE 'column_equivalent_single%' ORDER BY stxname; + stxname | stxkind +-------------------------------+--------- + column_equivalent_single_case | {e} + column_equivalent_single_vgen | {e} +(2 rows) + +ANALYZE column_equivalent; +SELECT statistics_name, expr, n_distinct FROM pg_stats_ext_exprs +WHERE statistics_name LIKE 'column_equivalent_single%' ORDER BY statistics_name; + statistics_name | expr | n_distinct +-------------------------------+------------------------+------------ + column_equivalent_single_case | +| 100 + | CASE +| + | WHEN true THEN a +| + | ELSE NULL::integer+| + | END | + column_equivalent_single_vgen | g | 100 +(2 rows) + +DROP TABLE column_equivalent; -- basic test for statistics on expressions CREATE TABLE ab1 (a INTEGER, b INTEGER, c TIMESTAMP, d TIMESTAMPTZ); -- expression stats may be built on a single expression column @@ -3203,6 +3292,23 @@ SELECT * FROM check_estimated_rows('SELECT * FROM virtual_gen_stats WHERE d = 0 10 | 10 (1 row) +SELECT expr FROM pg_stats_ext_exprs + WHERE statistics_name = 'virtual_gen_stats_1' AND NOT inherited; + expr +--------------- + c + (3 * b) + d + (d - (2 * a)) +(4 rows) + +SELECT pg_get_statisticsobjdef(oid) FROM pg_statistic_ext + WHERE stxname = 'virtual_gen_stats_1'; + pg_get_statisticsobjdef +--------------------------------------------------------------------------------------------------------- + CREATE STATISTICS public.virtual_gen_stats_1 (mcv) ON c, (3 * b), d, (d - 2 * a) FROM virtual_gen_stats +(1 row) + -- univariate statistics on individual virtual generated columns DROP STATISTICS virtual_gen_stats_1; SELECT * FROM check_estimated_rows('SELECT * FROM virtual_gen_stats WHERE c = 0'); @@ -3743,12 +3849,12 @@ CREATE STATISTICS stats_ext_range (mcv) ON irange, (irange + '[4,10)'::int4range) FROM stats_ext_tbl_range; ANALYZE stats_ext_tbl_range; -SELECT attnames, most_common_vals +SELECT exprs, most_common_vals FROM pg_stats_ext WHERE statistics_name = 'stats_ext_range'; - attnames | most_common_vals -----------+------------------------------------------------------------ - {irange} | {{"[1,7)","[1,10)"},{"[2,9)","[2,10)"},{"[3,9)","[3,10)"}} + exprs | most_common_vals +-------------------------------------------+------------------------------------------------------------ + {irange,"(irange + '[4,10)'::int4range)"} | {{"[1,7)","[1,10)"},{"[2,9)","[2,10)"},{"[3,9)","[3,10)"}} (1 row) SELECT range_length_histogram, range_empty_frac, range_bounds_histogram @@ -3778,3 +3884,82 @@ SELECT * FROM stats_ext_tbl_mcelem WHERE (a || a) @> '{1}'; (0 rows) DROP TABLE stats_ext_tbl_mcelem; +-- statistics objects with entries that const-fold to a plain column, left +-- undropped for pg_upgrade testing +CREATE TABLE column_equivalent_kept (a int, b int, g int GENERATED ALWAYS AS (a) VIRTUAL) WITH (autovacuum_enabled = off); +INSERT INTO column_equivalent_kept (a, b) SELECT i % 100, i % 10 FROM generate_series(1, 1000) i; +CREATE STATISTICS column_equivalent_kept_middle ON (a + b), (CASE WHEN true THEN a ELSE b END), (a * b) FROM column_equivalent_kept; +CREATE STATISTICS column_equivalent_kept_vgen ON g, b FROM column_equivalent_kept; +CREATE STATISTICS column_equivalent_kept_dup ON a, (CASE WHEN true THEN a END), b FROM column_equivalent_kept; +ANALYZE column_equivalent_kept; +SELECT stxname, pg_get_statisticsobjdef(oid) FROM pg_statistic_ext +WHERE stxname LIKE 'column_equivalent_kept%' ORDER BY stxname; + stxname | pg_get_statisticsobjdef +-------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------- + column_equivalent_kept_dup | CREATE STATISTICS public.column_equivalent_kept_dup ON a, (CASE WHEN true THEN a ELSE NULL::integer END), b FROM column_equivalent_kept + column_equivalent_kept_middle | CREATE STATISTICS public.column_equivalent_kept_middle ON (a + b), (CASE WHEN true THEN a ELSE b END), (a * b) FROM column_equivalent_kept + column_equivalent_kept_vgen | CREATE STATISTICS public.column_equivalent_kept_vgen ON g, b FROM column_equivalent_kept +(3 rows) + +-- the order of columns and expressions in CREATE STATISTICS is preserved in the +-- object definition and the MCV; left undropped for pg_upgrade testing +CREATE TABLE declared_order_preserved (a int, c int) WITH (autovacuum_enabled = off); +INSERT INTO declared_order_preserved SELECT 1, 100 FROM generate_series(1, 100) g; +CREATE STATISTICS declared_order_preserved_stat ON c, (a + c), a FROM declared_order_preserved; +ANALYZE declared_order_preserved; +SELECT pg_get_statisticsobjdef(oid) FROM pg_statistic_ext + WHERE stxname = 'declared_order_preserved_stat'; + pg_get_statisticsobjdef +------------------------------------------------------------------------------------------------------- + CREATE STATISTICS public.declared_order_preserved_stat ON c, (a + c), a FROM declared_order_preserved +(1 row) + +-- MCV values in CREATE STATISTICS order: c = 100, (a + c) = 101, a = 1 +SELECT m.values +FROM pg_statistic_ext s +JOIN pg_statistic_ext_data d ON (s.oid = d.stxoid) +CROSS JOIN LATERAL pg_mcv_list_items(d.stxdmcv) m +WHERE s.stxname = 'declared_order_preserved_stat'; + values +------------- + {100,101,1} +(1 row) + +-- selectivity estimates are accurate regardless of the column order in CREATE +-- STATISTICS (here b, a) +CREATE TABLE declared_order_selectivity (a int, b int) WITH (autovacuum_enabled = off); +-- correlated data: (a, b) is either (0, 1) or (1, 0) +INSERT INTO declared_order_selectivity SELECT 0, 1 FROM generate_series(1, 9900); +INSERT INTO declared_order_selectivity SELECT 1, 0 FROM generate_series(1, 100); +CREATE STATISTICS declared_order_selectivity_stat ON b, a FROM declared_order_selectivity; +ANALYZE declared_order_selectivity; +-- estimate should track the actual 9900 rows +SELECT * FROM check_estimated_rows('SELECT * FROM declared_order_selectivity WHERE a = 0 AND b = 1'); + estimated | actual +-----------+-------- + 9900 | 9900 +(1 row) + +-- estimate should track the actual 100 rows +SELECT * FROM check_estimated_rows('SELECT * FROM declared_order_selectivity WHERE a = 1 AND b = 0'); + estimated | actual +-----------+-------- + 100 | 100 +(1 row) + +DROP TABLE declared_order_selectivity; +-- grouping estimates are accurate regardless of the column order in CREATE +-- STATISTICS (here b, a) +CREATE TABLE declared_order_grouping (a int, b int) WITH (autovacuum_enabled = off); +-- correlated data: a and b move together, so 100 distinct (a, b) groups +INSERT INTO declared_order_grouping SELECT i%100, i%100 FROM generate_series(1, 10000) i; +CREATE STATISTICS declared_order_grouping_stat ON b, a FROM declared_order_grouping; +ANALYZE declared_order_grouping; +-- estimate should track the 100 distinct groups, not ndistinct(a) * ndistinct(b) +SELECT * FROM check_estimated_rows('SELECT COUNT(*) FROM declared_order_grouping GROUP BY a, b'); + estimated | actual +-----------+-------- + 100 | 100 +(1 row) + +DROP TABLE declared_order_grouping; diff --git a/src/test/regress/expected/stats_import.out b/src/test/regress/expected/stats_import.out index 004f35b8ad7..0702ae3e464 100644 --- a/src/test/regress/expected/stats_import.out +++ b/src/test/regress/expected/stats_import.out @@ -78,7 +78,7 @@ SELECT COUNT(*) FROM pg_attribute attnum > 0; count ------- - 17 + 16 (1 row) -- Create a view that is used purely for the type based on pg_stats_ext. @@ -4155,6 +4155,119 @@ FROM stats_import.pg_stats_ext_exprs_get_difference('test_mr_stat', 'test_mr_sta \gx (0 rows) +-- import of extended statistics that include a virtual generated column +CREATE TABLE stats_import.test_virtual_gen( + a int, + b int, + c int GENERATED ALWAYS AS (a + b) VIRTUAL +); +INSERT INTO stats_import.test_virtual_gen(a, b) + SELECT mod(i, 10), mod(i, 7) FROM generate_series(1, 1000) s(i); +CREATE STATISTICS stats_import.test_virtual_gen_stat ON a, c + FROM stats_import.test_virtual_gen; +ANALYZE stats_import.test_virtual_gen; +CREATE TABLE stats_import.test_virtual_gen_clone( + a int, + b int, + c int GENERATED ALWAYS AS (a + b) VIRTUAL +); +CREATE STATISTICS stats_import.test_virtual_gen_stat_clone ON a, c + FROM stats_import.test_virtual_gen_clone; +-- Import stats from test_virtual_gen_stat to test_virtual_gen_stat_clone +SELECT e.statistics_name, + pg_catalog.pg_restore_extended_stats( + 'schemaname', e.statistics_schemaname::text, + 'relname', 'test_virtual_gen_clone', + 'statistics_schemaname', e.statistics_schemaname::text, + 'statistics_name', 'test_virtual_gen_stat_clone', + 'inherited', e.inherited, + 'n_distinct', e.n_distinct, + 'dependencies', e.dependencies, + 'most_common_vals', e.most_common_vals, + 'most_common_freqs', e.most_common_freqs, + 'most_common_base_freqs', e.most_common_base_freqs, + 'exprs', x.exprs) +FROM pg_stats_ext AS e +CROSS JOIN LATERAL ( + SELECT jsonb_agg(jsonb_strip_nulls(jsonb_build_object( + 'null_frac', ee.null_frac::text, + 'avg_width', ee.avg_width::text, + 'n_distinct', ee.n_distinct::text, + 'most_common_vals', ee.most_common_vals::text, + 'most_common_freqs', ee.most_common_freqs::text, + 'histogram_bounds', ee.histogram_bounds::text, + 'correlation', ee.correlation::text))) + FROM pg_stats_ext_exprs AS ee + WHERE ee.statistics_schemaname = e.statistics_schemaname AND + ee.statistics_name = e.statistics_name AND + ee.inherited = e.inherited + ) AS x(exprs) +WHERE e.statistics_schemaname = 'stats_import' +AND e.statistics_name = 'test_virtual_gen_stat'; + statistics_name | pg_restore_extended_stats +-----------------------+--------------------------- + test_virtual_gen_stat | t +(1 row) + +SELECT statname, (stats).* +FROM stats_import.pg_stats_ext_get_difference('test_virtual_gen_stat', 'test_virtual_gen_stat_clone') +\gx +(0 rows) + +SELECT statname, (stats).* +FROM stats_import.pg_stats_ext_exprs_get_difference('test_virtual_gen_stat', 'test_virtual_gen_stat_clone') +\gx +(0 rows) + +-- round-trip an MCV that interleaves a column and an expression of different types +CREATE STATISTICS stats_import.test_stat_interleave (mcv) + ON name, lower(arange), comp + FROM stats_import.test; +CREATE STATISTICS stats_import.test_stat_interleave_clone (mcv) + ON name, lower(arange), comp + FROM stats_import.test_clone; +ANALYZE stats_import.test; +-- Copy stats from test_stat_interleave to test_stat_interleave_clone +SELECT e.statistics_name, + pg_catalog.pg_restore_extended_stats( + 'schemaname', e.statistics_schemaname::text, + 'relname', 'test_clone', + 'statistics_schemaname', e.statistics_schemaname::text, + 'statistics_name', 'test_stat_interleave_clone', + 'inherited', e.inherited, + 'n_distinct', e.n_distinct, + 'dependencies', e.dependencies, + 'most_common_vals', e.most_common_vals, + 'most_common_freqs', e.most_common_freqs, + 'most_common_base_freqs', e.most_common_base_freqs, + 'exprs', x.exprs) +FROM pg_stats_ext AS e +CROSS JOIN LATERAL ( + SELECT jsonb_agg(jsonb_strip_nulls(jsonb_build_object( + 'null_frac', ee.null_frac::text, + 'avg_width', ee.avg_width::text, + 'n_distinct', ee.n_distinct::text, + 'most_common_vals', ee.most_common_vals::text, + 'most_common_freqs', ee.most_common_freqs::text, + 'histogram_bounds', ee.histogram_bounds::text, + 'correlation', ee.correlation::text))) + FROM pg_stats_ext_exprs AS ee + WHERE ee.statistics_schemaname = e.statistics_schemaname AND + ee.statistics_name = e.statistics_name AND + ee.inherited = e.inherited + ) AS x(exprs) +WHERE e.statistics_schemaname = 'stats_import' +AND e.statistics_name = 'test_stat_interleave'; + statistics_name | pg_restore_extended_stats +----------------------+--------------------------- + test_stat_interleave | t +(1 row) + +SELECT statname, (stats).* +FROM stats_import.pg_stats_ext_get_difference('test_stat_interleave', 'test_stat_interleave_clone') +\gx +(0 rows) + -- range_length_histogram, range_empty_frac, and range_bounds_histogram -- have been added to pg_stats_ext_exprs in PostgreSQL 19. When dumping -- expression statistics in a cluster with an older version, these fields @@ -4482,8 +4595,49 @@ WARNING: "most_common_elems" must be an empty or one-dimensional array f (1 row) +-- Statistics data from a server before version 20, for an object with an +-- entry that const-folds to a plain column. Those versions number such an +-- entry among the expressions, here as -2, with its own element in "exprs". +CREATE TABLE stats_import.test_fold (a int, b int); +CREATE STATISTICS stats_import.test_fold_stat + ON (a + b), (CASE WHEN true THEN a ELSE b END) FROM stats_import.test_fold; +SELECT pg_catalog.pg_restore_extended_stats( + 'schemaname', 'stats_import', + 'relname', 'test_fold', + 'statistics_schemaname', 'stats_import', + 'statistics_name', 'test_fold_stat', + 'inherited', false, + 'version', 190000::integer, + 'n_distinct', '[{"attributes": [-1, -2], "ndistinct": 4}]'::pg_ndistinct, + 'exprs', '[{"null_frac": "0", "avg_width": "4", "n_distinct": "3"}, + {"null_frac": "0", "avg_width": "4", "n_distinct": "2"}]'::jsonb); + pg_restore_extended_stats +--------------------------- + t +(1 row) + +SELECT n_distinct FROM pg_stats_ext +WHERE statistics_schemaname = 'stats_import' AND statistics_name = 'test_fold_stat'; + n_distinct +-------------------------------------------- + [{"attributes": [-1, -2], "ndistinct": 4}] +(1 row) + +SELECT expr, n_distinct FROM pg_stats_ext_exprs +WHERE statistics_schemaname = 'stats_import' AND statistics_name = 'test_fold_stat' +ORDER BY expr; + expr | n_distinct +----------------------+------------ + +| 2 + CASE +| + WHEN true THEN a+| + ELSE b +| + END | + (a + b) | 3 +(2 rows) + DROP SCHEMA stats_import CASCADE; -NOTICE: drop cascades to 39 other objects +NOTICE: drop cascades to 42 other objects DETAIL: drop cascades to view stats_import.pg_stats_stable drop cascades to view stats_import.pg_statistic_flat_t drop cascades to function stats_import.pg_statistic_flat(text) @@ -4510,6 +4664,8 @@ drop cascades to table stats_import.test_dom_ts drop cascades to table stats_import.test_dom_ts_clone drop cascades to table stats_import.test_clone drop cascades to table stats_import.test_mr_clone +drop cascades to table stats_import.test_virtual_gen +drop cascades to table stats_import.test_virtual_gen_clone drop cascades to table stats_import.test_range_expr_null drop cascades to table stats_import.test_reentrant drop cascades to function stats_import.alter_during_import(uuid) @@ -4523,3 +4679,4 @@ drop cascades to type stats_import.reentrant_dom_ext2 drop cascades to table stats_import.test_reentrant_attr2 drop cascades to function stats_import.drop_during_import_attr(uuid) drop cascades to type stats_import.reentrant_dom_attr2 +drop cascades to table stats_import.test_fold diff --git a/src/test/regress/sql/create_table_like.sql b/src/test/regress/sql/create_table_like.sql index 5dd81f4eaf9..c66b6deb02e 100644 --- a/src/test/regress/sql/create_table_like.sql +++ b/src/test/regress/sql/create_table_like.sql @@ -292,7 +292,7 @@ SELECT attname, attcompression FROM pg_attribute WHERE attrelid = 'ctl_foreign_table2'::regclass and attnum > 0 ORDER BY attnum; -- LIKE ... INCLUDING STATISTICS with dropped columns in the parent, --- so stxkeys attnums are not contiguous. +-- so column attnums are not contiguous. CREATE TABLE ctl_stats3_parent (a int, b int, c int); ALTER TABLE ctl_stats3_parent DROP COLUMN b; CREATE STATISTICS ctl_stats3_stat ON a, c FROM ctl_stats3_parent; @@ -302,15 +302,10 @@ ALTER TABLE ctl_stats4_parent DROP COLUMN b; CREATE STATISTICS ctl_stats4_stat ON a, c FROM ctl_stats4_parent; CREATE TABLE ctl_stats4_child (LIKE ctl_stats4_parent INCLUDING STATISTICS); SELECT s.stxrelid::regclass AS relation, - array_agg(a.attname ORDER BY u.ord) AS stats_columns + pg_get_statisticsobjdef_columns(s.oid) AS stats_columns FROM pg_statistic_ext s -CROSS JOIN LATERAL - unnest(s.stxkeys::int2[]) WITH ORDINALITY AS u(attnum, ord) -JOIN pg_attribute a - ON a.attrelid = s.stxrelid AND a.attnum = u.attnum WHERE s.stxrelid IN ('ctl_stats3_child'::regclass, 'ctl_stats4_child'::regclass) -GROUP BY s.stxrelid ORDER BY s.stxrelid::regclass::text; DROP TABLE ctl_stats3_parent; DROP TABLE ctl_stats3_child; diff --git a/src/test/regress/sql/stats_ext.sql b/src/test/regress/sql/stats_ext.sql index 6045cfa51e8..bad72bcba43 100644 --- a/src/test/regress/sql/stats_ext.sql +++ b/src/test/regress/sql/stats_ext.sql @@ -196,6 +196,47 @@ SELECT * FROM check_estimated_rows('SELECT a, b FROM stxdinp GROUP BY 1, 2'); SELECT * FROM check_estimated_rows('SELECT a + 1, b FROM ONLY stxdinp GROUP BY 1, 2'); DROP TABLE stxdinp; +-- a column-equivalent expression (one that const-folds to a plain column, or a +-- passthrough virtual generated column) is still an expression, not that column +CREATE TABLE column_equivalent (a int, b int, g int GENERATED ALWAYS AS (a) VIRTUAL) WITH (autovacuum_enabled = off); +INSERT INTO column_equivalent (a, b) SELECT i%100, i%10 FROM generate_series(1, 10000) i; +-- a column-equivalent CASE between two complex expressions +CREATE STATISTICS column_equivalent_middle ON (a + b), (CASE WHEN true THEN a ELSE b END), (a * b) FROM column_equivalent; +-- only column-equivalent entries, which all fold to plain columns +CREATE STATISTICS column_equivalent_all ON (CASE WHEN true THEN a END), b FROM column_equivalent; +ANALYZE column_equivalent; +-- a column-equivalent CASE is reported as an expression, with its own +-- statistics, like the other expressions +SELECT statistics_name, expr, n_distinct +FROM pg_stats_ext_exprs +WHERE statistics_name IN ('column_equivalent_middle', 'column_equivalent_all') +ORDER BY statistics_name, expr; +-- each complex expression's GROUP BY estimate uses its own statistics +SELECT * FROM check_estimated_rows('SELECT (a + b) FROM column_equivalent GROUP BY 1'); +SELECT * FROM check_estimated_rows('SELECT (a * b) FROM column_equivalent GROUP BY 1'); +-- column_equivalent_all is not used for a GROUP BY on the plain columns a, b, +-- as its first entry is an expression rather than column a +SELECT * FROM check_estimated_rows('SELECT COUNT(*) FROM column_equivalent GROUP BY a, b'); +-- a parenthesized column is the same column, so it is rejected as a duplicate +CREATE STATISTICS column_equivalent_dup_paren ON a, (a) FROM column_equivalent; +-- but an expression that const-folds to a column already listed is not a +-- duplicate, +CREATE STATISTICS column_equivalent_with_case ON a, (CASE WHEN true THEN a END), b FROM column_equivalent; +-- and neither is a passthrough generated column +CREATE STATISTICS column_equivalent_with_vgen ON a, g FROM column_equivalent; +-- a generated column listed twice is a duplicate expression +CREATE STATISTICS column_equivalent_dup_vgen ON g, g FROM column_equivalent; +-- a single entry that const-folds to a plain column, and a single passthrough +-- generated column; each is an expression, with its own statistics +CREATE STATISTICS column_equivalent_single_case ON (CASE WHEN true THEN a END) FROM column_equivalent; +CREATE STATISTICS column_equivalent_single_vgen ON g FROM column_equivalent; +SELECT stxname, stxkind FROM pg_statistic_ext +WHERE stxname LIKE 'column_equivalent_single%' ORDER BY stxname; +ANALYZE column_equivalent; +SELECT statistics_name, expr, n_distinct FROM pg_stats_ext_exprs +WHERE statistics_name LIKE 'column_equivalent_single%' ORDER BY statistics_name; +DROP TABLE column_equivalent; + -- basic test for statistics on expressions CREATE TABLE ab1 (a INTEGER, b INTEGER, c TIMESTAMP, d TIMESTAMPTZ); @@ -1611,6 +1652,12 @@ ANALYZE virtual_gen_stats; SELECT * FROM check_estimated_rows('SELECT * FROM virtual_gen_stats WHERE c = 0 AND (3*b) = 0'); SELECT * FROM check_estimated_rows('SELECT * FROM virtual_gen_stats WHERE d = 0 AND (d-2*a) = 0'); +SELECT expr FROM pg_stats_ext_exprs + WHERE statistics_name = 'virtual_gen_stats_1' AND NOT inherited; + +SELECT pg_get_statisticsobjdef(oid) FROM pg_statistic_ext + WHERE stxname = 'virtual_gen_stats_1'; + -- univariate statistics on individual virtual generated columns DROP STATISTICS virtual_gen_stats_1; @@ -1937,7 +1984,7 @@ CREATE STATISTICS stats_ext_range (mcv) ON irange, (irange + '[4,10)'::int4range) FROM stats_ext_tbl_range; ANALYZE stats_ext_tbl_range; -SELECT attnames, most_common_vals +SELECT exprs, most_common_vals FROM pg_stats_ext WHERE statistics_name = 'stats_ext_range'; SELECT range_length_histogram, range_empty_frac, range_bounds_histogram @@ -1954,3 +2001,53 @@ SELECT most_common_elems, most_common_elem_freqs FROM pg_stats_ext_exprs WHERE statistics_name = 'stats_ext_mcelem'; SELECT * FROM stats_ext_tbl_mcelem WHERE (a || a) @> '{1}'; DROP TABLE stats_ext_tbl_mcelem; +-- statistics objects with entries that const-fold to a plain column, left +-- undropped for pg_upgrade testing +CREATE TABLE column_equivalent_kept (a int, b int, g int GENERATED ALWAYS AS (a) VIRTUAL) WITH (autovacuum_enabled = off); +INSERT INTO column_equivalent_kept (a, b) SELECT i % 100, i % 10 FROM generate_series(1, 1000) i; +CREATE STATISTICS column_equivalent_kept_middle ON (a + b), (CASE WHEN true THEN a ELSE b END), (a * b) FROM column_equivalent_kept; +CREATE STATISTICS column_equivalent_kept_vgen ON g, b FROM column_equivalent_kept; +CREATE STATISTICS column_equivalent_kept_dup ON a, (CASE WHEN true THEN a END), b FROM column_equivalent_kept; +ANALYZE column_equivalent_kept; +SELECT stxname, pg_get_statisticsobjdef(oid) FROM pg_statistic_ext +WHERE stxname LIKE 'column_equivalent_kept%' ORDER BY stxname; + +-- the order of columns and expressions in CREATE STATISTICS is preserved in the +-- object definition and the MCV; left undropped for pg_upgrade testing +CREATE TABLE declared_order_preserved (a int, c int) WITH (autovacuum_enabled = off); +INSERT INTO declared_order_preserved SELECT 1, 100 FROM generate_series(1, 100) g; +CREATE STATISTICS declared_order_preserved_stat ON c, (a + c), a FROM declared_order_preserved; +ANALYZE declared_order_preserved; +SELECT pg_get_statisticsobjdef(oid) FROM pg_statistic_ext + WHERE stxname = 'declared_order_preserved_stat'; +-- MCV values in CREATE STATISTICS order: c = 100, (a + c) = 101, a = 1 +SELECT m.values +FROM pg_statistic_ext s +JOIN pg_statistic_ext_data d ON (s.oid = d.stxoid) +CROSS JOIN LATERAL pg_mcv_list_items(d.stxdmcv) m +WHERE s.stxname = 'declared_order_preserved_stat'; + +-- selectivity estimates are accurate regardless of the column order in CREATE +-- STATISTICS (here b, a) +CREATE TABLE declared_order_selectivity (a int, b int) WITH (autovacuum_enabled = off); +-- correlated data: (a, b) is either (0, 1) or (1, 0) +INSERT INTO declared_order_selectivity SELECT 0, 1 FROM generate_series(1, 9900); +INSERT INTO declared_order_selectivity SELECT 1, 0 FROM generate_series(1, 100); +CREATE STATISTICS declared_order_selectivity_stat ON b, a FROM declared_order_selectivity; +ANALYZE declared_order_selectivity; +-- estimate should track the actual 9900 rows +SELECT * FROM check_estimated_rows('SELECT * FROM declared_order_selectivity WHERE a = 0 AND b = 1'); +-- estimate should track the actual 100 rows +SELECT * FROM check_estimated_rows('SELECT * FROM declared_order_selectivity WHERE a = 1 AND b = 0'); +DROP TABLE declared_order_selectivity; + +-- grouping estimates are accurate regardless of the column order in CREATE +-- STATISTICS (here b, a) +CREATE TABLE declared_order_grouping (a int, b int) WITH (autovacuum_enabled = off); +-- correlated data: a and b move together, so 100 distinct (a, b) groups +INSERT INTO declared_order_grouping SELECT i%100, i%100 FROM generate_series(1, 10000) i; +CREATE STATISTICS declared_order_grouping_stat ON b, a FROM declared_order_grouping; +ANALYZE declared_order_grouping; +-- estimate should track the 100 distinct groups, not ndistinct(a) * ndistinct(b) +SELECT * FROM check_estimated_rows('SELECT COUNT(*) FROM declared_order_grouping GROUP BY a, b'); +DROP TABLE declared_order_grouping; diff --git a/src/test/regress/sql/stats_import.sql b/src/test/regress/sql/stats_import.sql index fc50da45513..2f0c5885fe5 100644 --- a/src/test/regress/sql/stats_import.sql +++ b/src/test/regress/sql/stats_import.sql @@ -2900,6 +2900,108 @@ SELECT statname, (stats).* FROM stats_import.pg_stats_ext_exprs_get_difference('test_mr_stat', 'test_mr_stat_clone') \gx +-- import of extended statistics that include a virtual generated column +CREATE TABLE stats_import.test_virtual_gen( + a int, + b int, + c int GENERATED ALWAYS AS (a + b) VIRTUAL +); +INSERT INTO stats_import.test_virtual_gen(a, b) + SELECT mod(i, 10), mod(i, 7) FROM generate_series(1, 1000) s(i); +CREATE STATISTICS stats_import.test_virtual_gen_stat ON a, c + FROM stats_import.test_virtual_gen; +ANALYZE stats_import.test_virtual_gen; +CREATE TABLE stats_import.test_virtual_gen_clone( + a int, + b int, + c int GENERATED ALWAYS AS (a + b) VIRTUAL +); +CREATE STATISTICS stats_import.test_virtual_gen_stat_clone ON a, c + FROM stats_import.test_virtual_gen_clone; +-- Import stats from test_virtual_gen_stat to test_virtual_gen_stat_clone +SELECT e.statistics_name, + pg_catalog.pg_restore_extended_stats( + 'schemaname', e.statistics_schemaname::text, + 'relname', 'test_virtual_gen_clone', + 'statistics_schemaname', e.statistics_schemaname::text, + 'statistics_name', 'test_virtual_gen_stat_clone', + 'inherited', e.inherited, + 'n_distinct', e.n_distinct, + 'dependencies', e.dependencies, + 'most_common_vals', e.most_common_vals, + 'most_common_freqs', e.most_common_freqs, + 'most_common_base_freqs', e.most_common_base_freqs, + 'exprs', x.exprs) +FROM pg_stats_ext AS e +CROSS JOIN LATERAL ( + SELECT jsonb_agg(jsonb_strip_nulls(jsonb_build_object( + 'null_frac', ee.null_frac::text, + 'avg_width', ee.avg_width::text, + 'n_distinct', ee.n_distinct::text, + 'most_common_vals', ee.most_common_vals::text, + 'most_common_freqs', ee.most_common_freqs::text, + 'histogram_bounds', ee.histogram_bounds::text, + 'correlation', ee.correlation::text))) + FROM pg_stats_ext_exprs AS ee + WHERE ee.statistics_schemaname = e.statistics_schemaname AND + ee.statistics_name = e.statistics_name AND + ee.inherited = e.inherited + ) AS x(exprs) +WHERE e.statistics_schemaname = 'stats_import' +AND e.statistics_name = 'test_virtual_gen_stat'; + +SELECT statname, (stats).* +FROM stats_import.pg_stats_ext_get_difference('test_virtual_gen_stat', 'test_virtual_gen_stat_clone') +\gx + +SELECT statname, (stats).* +FROM stats_import.pg_stats_ext_exprs_get_difference('test_virtual_gen_stat', 'test_virtual_gen_stat_clone') +\gx + +-- round-trip an MCV that interleaves a column and an expression of different types +CREATE STATISTICS stats_import.test_stat_interleave (mcv) + ON name, lower(arange), comp + FROM stats_import.test; +CREATE STATISTICS stats_import.test_stat_interleave_clone (mcv) + ON name, lower(arange), comp + FROM stats_import.test_clone; +ANALYZE stats_import.test; +-- Copy stats from test_stat_interleave to test_stat_interleave_clone +SELECT e.statistics_name, + pg_catalog.pg_restore_extended_stats( + 'schemaname', e.statistics_schemaname::text, + 'relname', 'test_clone', + 'statistics_schemaname', e.statistics_schemaname::text, + 'statistics_name', 'test_stat_interleave_clone', + 'inherited', e.inherited, + 'n_distinct', e.n_distinct, + 'dependencies', e.dependencies, + 'most_common_vals', e.most_common_vals, + 'most_common_freqs', e.most_common_freqs, + 'most_common_base_freqs', e.most_common_base_freqs, + 'exprs', x.exprs) +FROM pg_stats_ext AS e +CROSS JOIN LATERAL ( + SELECT jsonb_agg(jsonb_strip_nulls(jsonb_build_object( + 'null_frac', ee.null_frac::text, + 'avg_width', ee.avg_width::text, + 'n_distinct', ee.n_distinct::text, + 'most_common_vals', ee.most_common_vals::text, + 'most_common_freqs', ee.most_common_freqs::text, + 'histogram_bounds', ee.histogram_bounds::text, + 'correlation', ee.correlation::text))) + FROM pg_stats_ext_exprs AS ee + WHERE ee.statistics_schemaname = e.statistics_schemaname AND + ee.statistics_name = e.statistics_name AND + ee.inherited = e.inherited + ) AS x(exprs) +WHERE e.statistics_schemaname = 'stats_import' +AND e.statistics_name = 'test_stat_interleave'; + +SELECT statname, (stats).* +FROM stats_import.pg_stats_ext_get_difference('test_stat_interleave', 'test_stat_interleave_clone') +\gx + -- range_length_histogram, range_empty_frac, and range_bounds_histogram -- have been added to pg_stats_ext_exprs in PostgreSQL 19. When dumping -- expression statistics in a cluster with an older version, these fields @@ -3163,5 +3265,26 @@ SELECT pg_catalog.pg_restore_attribute_stats( 'inherited', false::boolean, 'most_common_elems', '{{one},{two}}'::text, 'most_common_elem_freqs', '{0.1,0.1,0.1,0.1,0}'::real[]); +-- Statistics data from a server before version 20, for an object with an +-- entry that const-folds to a plain column. Those versions number such an +-- entry among the expressions, here as -2, with its own element in "exprs". +CREATE TABLE stats_import.test_fold (a int, b int); +CREATE STATISTICS stats_import.test_fold_stat + ON (a + b), (CASE WHEN true THEN a ELSE b END) FROM stats_import.test_fold; +SELECT pg_catalog.pg_restore_extended_stats( + 'schemaname', 'stats_import', + 'relname', 'test_fold', + 'statistics_schemaname', 'stats_import', + 'statistics_name', 'test_fold_stat', + 'inherited', false, + 'version', 190000::integer, + 'n_distinct', '[{"attributes": [-1, -2], "ndistinct": 4}]'::pg_ndistinct, + 'exprs', '[{"null_frac": "0", "avg_width": "4", "n_distinct": "3"}, + {"null_frac": "0", "avg_width": "4", "n_distinct": "2"}]'::jsonb); +SELECT n_distinct FROM pg_stats_ext +WHERE statistics_schemaname = 'stats_import' AND statistics_name = 'test_fold_stat'; +SELECT expr, n_distinct FROM pg_stats_ext_exprs +WHERE statistics_schemaname = 'stats_import' AND statistics_name = 'test_fold_stat' +ORDER BY expr; DROP SCHEMA stats_import CASCADE; -- 2.54.0 (Apple Git-157)