From 769275496aa8f94b2c27e2aa31dd3672667b3eca Mon Sep 17 00:00:00 2001 From: Alexandra Wang Date: Thu, 16 Apr 2026 11:20:27 -0700 Subject: [PATCH v1] Add join MCV statistics for selectivity estimation Extends the extended statistics framework to support MCV lists that span an equijoin between two tables. This lets the planner use per-value join frequencies instead of uniform estimates (1/ndistinct or 1/ref_tuples) when filter predicates are present on the joined tables. Syntax: CREATE STATISTICS s (mcv) ON d.name FROM fact f JOIN dim d ON (f.dim_id = d.id); The pg_statistic_ext catalog table gains three nullable columns: stxjoinrels, stxkeyrefs, and stxjoinconds. They are only used by join statistics, and are set to NULL for single-table statistics. We call the first table in the FROM clause the "anchor" table (the fact table in the above example). During ANALYZE, the anchor table is sampled, and for each sampled row, the other table (dim table in above example) is probed via index lookup. Rows are weighted by match count to reproduce the join's row distribution. The result is a standard MCVList. Currently, only 2-way joins are collected; n-way stats are accepted at creation time, but skipped during ANALYZE with a warning. The index-based sampling approach is based on the technique described in "How Good Are Query Optimizers, Really?" (Leis et al., CIDR 2017). The planner consumes join MCVs at two points: regular join selectivity (clausesel.c) and FK-path joins (costsize.c). Covered filter predicates are folded into the MCV selectivity and divided out so the caller's per-clause loop does not double-count. DDL supports CREATE, DROP, and round-trip deparse via pg_get_statisticsobjdef(). A NORMAL dependency on the required index ensures the stats object blocks DROP INDEX (requires CASCADE). Limitations: - Single-condition equijoins between two tables only. - Only "col = const" and "col IN (...)" filter predicates recognized. - n-way join stats stored but not yet collected or used. Reference: https://www.cidrdb.org/cidr2017/papers/p9-leis-cidr17.pdf Discussion: https://www.postgresql.org/message-id/flat/CADkLM%3DcUwMftPLFq0iD6-qKRyNiRM2HZGYVp6%3D0noxA8GfuEtA%40mail.gmail.com --- src/backend/catalog/system_views.sql | 20 +- src/backend/commands/statscmds.c | 531 ++++- src/backend/optimizer/path/clausesel.c | 44 +- src/backend/optimizer/path/costsize.c | 32 +- src/backend/optimizer/util/plancat.c | 67 +- src/backend/parser/gram.y | 7 + src/backend/parser/parse_utilcmd.c | 211 +- src/backend/statistics/Makefile | 1 + src/backend/statistics/extended_stats.c | 249 ++- src/backend/statistics/join_mcv.c | 1723 +++++++++++++++++ src/backend/statistics/meson.build | 1 + src/backend/tcop/utility.c | 52 +- src/backend/utils/adt/ruleutils.c | 200 +- src/bin/psql/describe.c | 59 +- src/include/catalog/catversion.h | 2 +- src/include/catalog/pg_proc.dat | 4 + src/include/catalog/pg_statistic_ext.h | 15 + src/include/nodes/parsenodes.h | 5 + src/include/nodes/pathnodes.h | 6 + .../statistics/extended_stats_internal.h | 19 +- src/include/statistics/statistics.h | 8 + src/test/regress/expected/oidjoins.out | 12 + src/test/regress/expected/rules.out | 16 +- src/test/regress/expected/stats_ext.out | 14 +- .../regress/expected/stats_ext_crossrel.out | 1312 +++++++++++++ src/test/regress/parallel_schedule | 3 + src/test/regress/sql/oidjoins.sql | 12 + src/test/regress/sql/stats_ext_crossrel.sql | 991 ++++++++++ 28 files changed, 5398 insertions(+), 218 deletions(-) create mode 100644 src/backend/statistics/join_mcv.c create mode 100644 src/test/regress/expected/stats_ext_crossrel.out create mode 100644 src/test/regress/sql/stats_ext_crossrel.sql diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 73a1c1c4670..8a7d360eae0 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -285,10 +285,17 @@ CREATE VIEW pg_stats_ext WITH (security_barrier) AS s.stxname AS statistics_name, s.oid AS statistics_id, pg_get_userbyid(s.stxowner) AS statistics_owner, - ( SELECT array_agg(a.attname ORDER BY a.attnum) - FROM unnest(s.stxkeys) k - JOIN pg_attribute a - ON (a.attrelid = s.stxrelid AND a.attnum = k) + ( SELECT array_agg(a.attname ORDER BY k.ord) + FROM unnest(s.stxkeys) WITH ORDINALITY AS k(attnum, ord) + LEFT JOIN unnest(s.stxkeyrefs) WITH ORDINALITY + AS r(keyref, ord2) ON (k.ord = r.ord2) + JOIN pg_attribute a ON ( + a.attrelid = CASE + WHEN r.keyref IS NULL OR r.keyref = 1 + THEN s.stxrelid + ELSE s.stxjoinrels[r.keyref - 2] + END + AND a.attnum = k.attnum) ) AS attnames, pg_get_statisticsobjdef_expressions(s.oid) as exprs, s.stxkind AS kinds, @@ -311,6 +318,11 @@ CREATE VIEW pg_stats_ext WITH (security_barrier) AS FROM pg_mcv_list_items(sd.stxdmcv) ) m ON sd.stxdmcv IS NOT NULL WHERE pg_has_role(c.relowner, 'USAGE') + AND (s.stxjoinrels IS NULL OR NOT EXISTS ( + SELECT 1 FROM unnest(s.stxjoinrels) AS jr(oid) + JOIN pg_class jc ON jc.oid = jr.oid + WHERE NOT pg_has_role(jc.relowner, 'USAGE') + )) AND (c.relrowsecurity = false OR NOT row_security_active(c.oid)); CREATE VIEW pg_stats_ext_exprs WITH (security_barrier) AS diff --git a/src/backend/commands/statscmds.c b/src/backend/commands/statscmds.c index b354723be44..f37cf4506c8 100644 --- a/src/backend/commands/statscmds.c +++ b/src/backend/commands/statscmds.c @@ -14,6 +14,7 @@ */ #include "postgres.h" +#include "access/genam.h" #include "access/htup_details.h" #include "access/relation.h" #include "access/table.h" @@ -31,6 +32,7 @@ #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "optimizer/optimizer.h" +#include "statistics/extended_stats_internal.h" #include "statistics/statistics.h" #include "utils/acl.h" #include "utils/builtins.h" @@ -79,7 +81,7 @@ CreateStatistics(CreateStatsStmt *stmt, bool check_rights) Datum exprsDatum; Relation statrel; Relation rel = NULL; - Oid relid; + Oid relid = InvalidOid; ObjectAddress parentobject, myself; Datum types[4]; /* one for each possible type of statistic */ @@ -93,29 +95,28 @@ CreateStatistics(CreateStatsStmt *stmt, bool check_rights) int i; ListCell *cell; ListCell *cell2; + ListCell *lc; + Node *rln; + bool isjoin = false; + List *other_rels = NIL; + int *key_varnos = NULL; /* stats key's varnos, for joins */ + List *join_idx_oids = NIL; Assert(IsA(stmt, CreateStatsStmt)); /* - * Examine the FROM clause. Currently, we only allow it to be a single - * simple table, but later we'll probably allow multiple tables and JOIN - * syntax. The grammar is already prepared for that, so we have to check - * here that what we got is what we can support. + * Examine the FROM clause. We support either: 1. Single RangeVar for + * single-table statistics 2. JoinExpr for join statistics */ if (list_length(stmt->relations) != 1) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("only a single relation is allowed in CREATE STATISTICS"))); - - foreach(cell, stmt->relations) - { - Node *rln = (Node *) lfirst(cell); + errmsg("only a single relation or JOIN is allowed in CREATE STATISTICS"))); - if (!IsA(rln, RangeVar)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("only a single relation is allowed in CREATE STATISTICS"))); + rln = (Node *) linitial(stmt->relations); + if (IsA(rln, RangeVar)) + { /* * CREATE STATISTICS will influence future execution plans but does * not interfere with currently executing plans. So it should be @@ -153,69 +154,20 @@ CreateStatistics(CreateStatsStmt *stmt, bool check_rights) (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("permission denied: \"%s\" is a system catalog", RelationGetRelationName(rel)))); - } - - Assert(rel); - relid = RelationGetRelid(rel); - /* - * If the node has a name, split it up and determine creation namespace. - * If not, put the object in the same namespace as the relation, and cons - * up a name for it. (This can happen either via "CREATE STATISTICS ..." - * or via "CREATE TABLE ... (LIKE)".) - */ - if (stmt->defnames) - namespaceId = QualifiedNameGetCreationNamespace(stmt->defnames, - &namestr); - else - { - namespaceId = RelationGetNamespace(rel); - namestr = ChooseExtendedStatisticName(RelationGetRelationName(rel), - ChooseExtendedStatisticNameAddition(stmt->exprs), - "stat", - namespaceId); + relid = RelationGetRelid(rel); } - namestrcpy(&stxname, namestr); - - /* - * Check we have creation rights in target namespace. Skip check if - * caller doesn't want it. - */ - if (check_rights) + else if (IsA(rln, JoinExpr)) { - AclResult aclresult; - - aclresult = object_aclcheck(NamespaceRelationId, namespaceId, - GetUserId(), ACL_CREATE); - if (aclresult != ACLCHECK_OK) - aclcheck_error(aclresult, OBJECT_SCHEMA, - get_namespace_name(namespaceId)); + Assert(stmt->transformed); + isjoin = true; + relid = stmt->stxrelid; } - - /* - * Deal with the possibility that the statistics object already exists. - */ - if (SearchSysCacheExists2(STATEXTNAMENSP, - CStringGetDatum(namestr), - ObjectIdGetDatum(namespaceId))) + else { - if (stmt->if_not_exists) - { - /* - * Since stats objects aren't members of extensions (see comments - * below), no need for checkMembershipInCurrentExtension here. - */ - ereport(NOTICE, - (errcode(ERRCODE_DUPLICATE_OBJECT), - errmsg("statistics object \"%s\" already exists, skipping", - namestr))); - relation_close(rel, NoLock); - return InvalidObjectAddress; - } - ereport(ERROR, - (errcode(ERRCODE_DUPLICATE_OBJECT), - errmsg("statistics object \"%s\" already exists", namestr))); + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("only a single relation or JOIN is allowed in CREATE STATISTICS"))); } /* @@ -254,6 +206,12 @@ CreateStatistics(CreateStatsStmt *stmt, bool check_rights) Form_pg_attribute attForm; TypeCacheEntry *type; + /* Join stats require table-qualified column names */ + if (isjoin) + ereport(ERROR, + (errcode(ERRCODE_INVALID_COLUMN_REFERENCE), + errmsg("join statistics require table-qualified column names"))); + attname = selem->name; atttuple = SearchSysCacheAttName(relid, attname); @@ -317,11 +275,8 @@ CreateStatistics(CreateStatsStmt *stmt, bool check_rights) (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("statistics creation on system columns is not supported"))); - /* - * Disallow data types without a less-than operator in - * multivariate statistics. - */ - if (numcols > 1) + /* For join stats, record which table each column comes from. */ + if (isjoin) { type = lookup_type_cache(var->vartype, TYPECACHE_LT_OPR); if (type->lt_opr == InvalidOid) @@ -331,17 +286,41 @@ CreateStatistics(CreateStatsStmt *stmt, bool check_rights) get_attname(relid, var->varattno, false)), errdetail("The type %s has no default btree operator class.", format_type_be(var->vartype)))); - } - /* Treat virtual generated columns as expressions */ - if (get_attgenerated(relid, var->varattno) == ATTRIBUTE_GENERATED_VIRTUAL) - { - stxexprs = lappend(stxexprs, (Node *) var); + /* Allocate key_varnos on first use */ + if (key_varnos == NULL) + key_varnos = palloc0(STATS_MAX_DIMENSIONS * sizeof(int)); + + key_varnos[nattnums] = var->varno; + attnums[nattnums] = var->varattno; + nattnums++; } else { - attnums[nattnums] = var->varattno; - nattnums++; + /* + * Disallow data types without a less-than operator in + * multivariate statistics. + */ + if (numcols > 1) + { + type = lookup_type_cache(var->vartype, TYPECACHE_LT_OPR); + if (type->lt_opr == InvalidOid) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("column \"%s\" cannot be used in multivariate statistics because its type %s has no default btree operator class", + get_attname(relid, var->varattno, false), format_type_be(var->vartype)))); + } + + /* Treat virtual generated columns as expressions */ + if (get_attgenerated(relid, var->varattno) == ATTRIBUTE_GENERATED_VIRTUAL) + { + stxexprs = lappend(stxexprs, (Node *) var); + } + else + { + attnums[nattnums] = var->varattno; + nattnums++; + } } } else /* expression */ @@ -354,6 +333,12 @@ CreateStatistics(CreateStatsStmt *stmt, bool check_rights) Assert(expr != NULL); + /* Join stats only support simple column references */ + if (isjoin) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("expressions are not supported in join statistics"))); + pull_varattnos(expr, 1, &attnums); k = -1; @@ -388,12 +373,204 @@ CreateStatistics(CreateStatsStmt *stmt, bool check_rights) } } + /* + * For join statistics, open and validate all participating relations. The + * join info (stxrelid, stxjoinrels, stxjoinconds) was already resolved + * and validated by transformStatsStmt(). + */ + if (isjoin) + { + /* Open anchor relation and validate */ + rel = relation_open(relid, NoLock); + + if (rel->rd_rel->relkind != RELKIND_RELATION && + rel->rd_rel->relkind != RELKIND_MATVIEW && + rel->rd_rel->relkind != RELKIND_FOREIGN_TABLE && + rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE) + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("cannot define statistics for relation \"%s\"", + RelationGetRelationName(rel)), + errdetail_relkind_not_supported(rel->rd_rel->relkind))); + if (!object_ownercheck(RelationRelationId, relid, stxowner)) + aclcheck_error(ACLCHECK_NOT_OWNER, + get_relkind_objtype(rel->rd_rel->relkind), + RelationGetRelationName(rel)); + if (!allowSystemTableMods && IsSystemRelation(rel)) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("permission denied: \"%s\" is a system catalog", + RelationGetRelationName(rel)))); + + /* Open all join relations and validate */ + foreach(lc, stmt->stxjoinrels) + { + Relation jrel = relation_open(lfirst_oid(lc), NoLock); + + if (jrel->rd_rel->relkind != RELKIND_RELATION && + jrel->rd_rel->relkind != RELKIND_MATVIEW && + jrel->rd_rel->relkind != RELKIND_FOREIGN_TABLE && + jrel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE) + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("cannot define statistics for relation \"%s\"", + RelationGetRelationName(jrel)), + errdetail_relkind_not_supported(jrel->rd_rel->relkind))); + if (!object_ownercheck(RelationRelationId, + RelationGetRelid(jrel), stxowner)) + aclcheck_error(ACLCHECK_NOT_OWNER, + get_relkind_objtype(jrel->rd_rel->relkind), + RelationGetRelationName(jrel)); + if (!allowSystemTableMods && IsSystemRelation(jrel)) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("permission denied: \"%s\" is a system catalog", + RelationGetRelationName(jrel)))); + + other_rels = lappend(other_rels, jrel); + } + + /* + * Validate that a suitable index exists for each join condition. + * + * Index-based sampling starts from the anchor table (varno 1) and + * probes the non-anchor side of each condition via index lookup, + * following the user-declared FROM clause order. The index must + * exist on the non-anchor side's join column. + */ + foreach(lc, stmt->stxjoinconds) + { + OpExpr *opexpr = (OpExpr *) lfirst(lc); + Var *lvar = (Var *) linitial(opexpr->args); + Var *rvar = (Var *) lsecond(opexpr->args); + Var *indexed_var; + Relation indexed_rel; + Oid idx_oid; + + /* + * Sampling probes the table with the higher varno (the one being + * joined into the growing sample) via index lookup. That side + * must have a suitable index. + */ + indexed_var = (lvar->varno > rvar->varno) ? lvar : rvar; + Assert(indexed_var->varno > 1); + indexed_rel = (Relation) list_nth(other_rels, + indexed_var->varno - 2); + + idx_oid = find_index_for_operator(indexed_rel, + indexed_var->varattno, + opexpr->opno); + if (!OidIsValid(idx_oid)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("no suitable index on \"%s\" column \"%s\" for join statistics", + RelationGetRelationName(indexed_rel), + get_attname(RelationGetRelid(indexed_rel), + indexed_var->varattno, false)), + errhint("Create an index on the join column to enable index-based join sampling."))); + + join_idx_oids = lappend_oid(join_idx_oids, idx_oid); + } + + /* + * Validate statistics columns. Each column's relation is determined + * by its parser varno: 1 = stxrelid, 2 = stxjoinrels[0], etc. + */ + for (i = 0; i < nattnums; i++) + { + TypeCacheEntry *type; + int varno = key_varnos[i]; + Oid key_relid; + Oid atttype; + + if (varno == 1) + key_relid = relid; + else + key_relid = list_nth_oid(stmt->stxjoinrels, varno - 2); + + if (get_attgenerated(key_relid, attnums[i]) == ATTRIBUTE_GENERATED_VIRTUAL) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("statistics creation on virtual generated columns is not supported"))); + + atttype = get_atttype(key_relid, attnums[i]); + type = lookup_type_cache(atttype, TYPECACHE_LT_OPR); + if (type->lt_opr == InvalidOid) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("column \"%s\" cannot be used in statistics because its type %s has no default btree operator class", + get_attname(key_relid, attnums[i], false), + format_type_be(atttype)))); + } + } + + /* + * Now determine namespace and name. Use rel (anchor table for joins). + */ + if (stmt->defnames) + namespaceId = QualifiedNameGetCreationNamespace(stmt->defnames, + &namestr); + else + { + namespaceId = RelationGetNamespace(rel); + namestr = ChooseExtendedStatisticName(RelationGetRelationName(rel), + ChooseExtendedStatisticNameAddition(stmt->exprs), + "stat", + namespaceId); + } + namestrcpy(&stxname, namestr); + + /* + * Check we have creation rights in target namespace. Skip check if + * caller doesn't want it. + */ + if (check_rights) + { + AclResult aclresult; + + aclresult = object_aclcheck(NamespaceRelationId, namespaceId, + GetUserId(), ACL_CREATE); + if (aclresult != ACLCHECK_OK) + aclcheck_error(aclresult, OBJECT_SCHEMA, + get_namespace_name(namespaceId)); + } + + /* + * Deal with the possibility that the statistics object already exists. + */ + if (SearchSysCacheExists2(STATEXTNAMENSP, + CStringGetDatum(namestr), + ObjectIdGetDatum(namespaceId))) + { + if (stmt->if_not_exists) + { + /* + * Since stats objects aren't members of extensions (see comments + * below), no need for checkMembershipInCurrentExtension here. + */ + ereport(NOTICE, + (errcode(ERRCODE_DUPLICATE_OBJECT), + errmsg("statistics object \"%s\" already exists, skipping", + namestr))); + /* Close relations */ + relation_close(rel, NoLock); + foreach(lc, other_rels) + relation_close((Relation) lfirst(lc), NoLock); + return InvalidObjectAddress; + } + + ereport(ERROR, + (errcode(ERRCODE_DUPLICATE_OBJECT), + errmsg("statistics object \"%s\" already exists", namestr))); + } + /* * Check that at least two columns were specified in the statement, or * that we're building statistics on a single expression (or virtual - * generated column). + * generated column). Join statistics are exempt since single-column + * MCV over a join captures cross-relation frequency data. */ - if (numcols < 2 && list_length(stxexprs) != 1) + if (numcols < 2 && list_length(stxexprs) != 1 && other_rels == NIL) ereport(ERROR, errcode(ERRCODE_INVALID_OBJECT_DEFINITION), errmsg("cannot create extended statistics on a single non-virtual column"), @@ -403,7 +580,7 @@ CreateStatistics(CreateStatsStmt *stmt, bool check_rights) * Parse the statistics kinds (not allowed when building univariate * statistics). */ - if (numcols == 1 && stmt->stat_types != NIL) + if (numcols == 1 && stmt->stat_types != NIL && other_rels == NIL) ereport(ERROR, errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot specify statistics kinds when building univariate statistics")); @@ -455,23 +632,61 @@ CreateStatistics(CreateStatsStmt *stmt, bool check_rights) */ build_expressions = (stxexprs != NIL); + /* + * Check column count requirements. + */ + if (isjoin) + { + /* Join stats need at least 1 column (join adds implicit correlation) */ + if (numcols < 1) + ereport(ERROR, + (errcode(ERRCODE_INVALID_OBJECT_DEFINITION), + errmsg("extended join statistics require at least one column"))); + } + else + { + /* Single-table stats need at least 2 columns or 1 expression */ + if (numcols < 2 && list_length(stxexprs) != 1) + ereport(ERROR, + (errcode(ERRCODE_INVALID_OBJECT_DEFINITION), + errmsg("extended statistics require at least 2 columns"))); + } + /* * Sort the attnums, which makes detecting duplicates somewhat easier, and * it does not hurt (it does not matter for the contents, unlike for * indexes, for example). + * + * For join stats, preserve user-declared column order and check for + * duplicate (varno, attnum) pairs. */ - qsort(attnums, nattnums, sizeof(int16), compare_int16); + if (isjoin) + { + int j; - /* - * Check for duplicates in the list of columns. The attnums are sorted so - * just check consecutive elements. - */ - for (i = 1; i < nattnums; i++) + for (i = 0; i < nattnums; i++) + { + for (j = i + 1; j < nattnums; j++) + { + if (attnums[i] == attnums[j] && + key_varnos[i] == key_varnos[j]) + ereport(ERROR, + (errcode(ERRCODE_DUPLICATE_COLUMN), + errmsg("duplicate column name in statistics definition"))); + } + } + } + else { - if (attnums[i] == attnums[i - 1]) - ereport(ERROR, - (errcode(ERRCODE_DUPLICATE_COLUMN), - errmsg("duplicate column name in statistics definition"))); + qsort(attnums, nattnums, sizeof(int16), compare_int16); + + for (i = 1; i < nattnums; i++) + { + if (attnums[i] == attnums[i - 1]) + ereport(ERROR, + (errcode(ERRCODE_DUPLICATE_COLUMN), + errmsg("duplicate column name in statistics definition"))); + } } /* @@ -512,6 +727,15 @@ CreateStatistics(CreateStatsStmt *stmt, bool check_rights) /* Form an int2vector representation of the sorted column list */ stxkeys = buildint2vector(attnums, nattnums); + /* + * For join statistics, only MCV is currently supported. + */ + if (isjoin && (build_ndistinct || build_dependencies || build_expressions)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("only MCV statistics are supported for join statistics"), + errhint("ndistinct, dependencies, and expression statistics require a single table."))); + /* construct the char array of enabled statistic types */ ntypes = 0; if (build_ndistinct) @@ -560,6 +784,46 @@ CreateStatistics(CreateStatsStmt *stmt, bool check_rights) if (exprsDatum == (Datum) 0) nulls[Anum_pg_statistic_ext_stxexprs - 1] = true; + /* + * For join statistics, populate stxjoinrels, stxkeyrefs, and + * stxjoinconds. For single-table statistics, these fields are NULL. + */ + if (isjoin) + { + int2vector *stxkeyrefs; + int16 keyrefs[STATS_MAX_DIMENSIONS]; + int nkeys = stxkeys->dim1; + int njoinrels = list_length(stmt->stxjoinrels); + Oid *joinrel_arr; + int idx; + + /* stxjoinrels: array of other relation OIDs */ + joinrel_arr = palloc(njoinrels * sizeof(Oid)); + idx = 0; + foreach(lc, stmt->stxjoinrels) + joinrel_arr[idx++] = lfirst_oid(lc); + values[Anum_pg_statistic_ext_stxjoinrels - 1] = + PointerGetDatum(buildoidvector(joinrel_arr, njoinrels)); + + /* stxkeyrefs: parser varno of each column's source relation */ + for (i = 0; i < nkeys; i++) + keyrefs[i] = key_varnos[i]; + stxkeyrefs = buildint2vector(keyrefs, nkeys); + values[Anum_pg_statistic_ext_stxkeyrefs - 1] = + PointerGetDatum(stxkeyrefs); + + /* stxjoinconds: serialize the join conditions */ + values[Anum_pg_statistic_ext_stxjoinconds - 1] = + CStringGetTextDatum(nodeToString(stmt->stxjoinconds)); + } + else + { + /* Join fields are NULL for single-table statistics */ + nulls[Anum_pg_statistic_ext_stxjoinrels - 1] = true; + nulls[Anum_pg_statistic_ext_stxkeyrefs - 1] = true; + nulls[Anum_pg_statistic_ext_stxjoinconds - 1] = true; + } + /* insert it into pg_statistic_ext */ htup = heap_form_tuple(statrel->rd_att, values, nulls); CatalogTupleInsert(statrel, htup); @@ -576,11 +840,16 @@ CreateStatistics(CreateStatsStmt *stmt, bool check_rights) InvokeObjectPostCreateHook(StatisticExtRelationId, statoid, 0); /* - * Invalidate relcache so that others see the new statistics object. + * Invalidate relcache so that others see the new statistics object. Only + * the anchor relation needs invalidation -- it's the one whose statlist + * is used to find this stats object during planning. */ CacheInvalidateRelcache(rel); + /* Close relations */ relation_close(rel, NoLock); + foreach(lc, other_rels) + relation_close((Relation) lfirst(lc), NoLock); /* * Add an AUTO dependency on each column used in the stats, so that the @@ -591,10 +860,71 @@ CreateStatistics(CreateStatsStmt *stmt, bool check_rights) /* add dependencies for plain column references */ for (i = 0; i < nattnums; i++) { - ObjectAddressSubSet(parentobject, RelationRelationId, relid, attnums[i]); + Oid key_relid; + + if (isjoin) + { + int varno = key_varnos[i]; + + if (varno == 1) + key_relid = relid; + else + key_relid = list_nth_oid(stmt->stxjoinrels, varno - 2); + } + else + key_relid = relid; + + ObjectAddressSubSet(parentobject, RelationRelationId, key_relid, attnums[i]); recordDependencyOn(&myself, &parentobject, DEPENDENCY_AUTO); } + /* For join stats, add dependencies on join columns */ + if (isjoin) + { + ListCell *lc; + Oid all_rels[STATS_MAX_DIMENSIONS + 1]; + int nrels; + + /* Build relation OID array: [stxrelid, joinrels...] */ + all_rels[0] = relid; + nrels = 1; + foreach(lc, stmt->stxjoinrels) + all_rels[nrels++] = lfirst_oid(lc); + + /* Record dependencies on columns referenced by join conditions */ + foreach(lc, stmt->stxjoinconds) + { + OpExpr *op = (OpExpr *) lfirst(lc); + ListCell *lc2; + + foreach(lc2, op->args) + { + Node *arg = (Node *) lfirst(lc2); + + if (IsA(arg, Var)) + { + Var *var = (Var *) arg; + + /* varno is 1-based; all_rels is 0-based */ + Assert(var->varno >= 1 && var->varno <= nrels); + ObjectAddressSubSet(parentobject, RelationRelationId, + all_rels[var->varno - 1], + var->varattno); + recordDependencyOn(&myself, &parentobject, + DEPENDENCY_AUTO); + } + } + } + + /* Record dependencies on indexes required for join sampling */ + foreach(lc, join_idx_oids) + { + ObjectAddressSet(parentobject, RelationRelationId, + lfirst_oid(lc)); + recordDependencyOn(&myself, &parentobject, DEPENDENCY_NORMAL); + } + } + /* * If there are no dependencies on a column, give the statistics object an * auto dependency on the whole table. In most cases, this will be @@ -605,8 +935,11 @@ CreateStatistics(CreateStatsStmt *stmt, bool check_rights) * XXX We intentionally don't consider the expressions before adding this * dependency, because recordDependencyOnSingleRelExpr may not create any * dependencies for whole-row Vars. + * + * Join stats are excluded because they span two tables (no single "whole + * table" exists). */ - if (!nattnums) + if (!isjoin && !nattnums) { ObjectAddressSet(parentobject, RelationRelationId, relid); recordDependencyOn(&myself, &parentobject, DEPENDENCY_AUTO); diff --git a/src/backend/optimizer/path/clausesel.c b/src/backend/optimizer/path/clausesel.c index 25c4d177ad9..ccfeaa82950 100644 --- a/src/backend/optimizer/path/clausesel.c +++ b/src/backend/optimizer/path/clausesel.c @@ -129,19 +129,45 @@ clauselist_selectivity_ext(PlannerInfo *root, int listidx; /* - * If there's exactly one clause, just go directly to - * clause_selectivity_ext(). None of what we might do below is relevant. + * Determine if these clauses reference a single relation. */ - if (list_length(clauses) == 1) + rel = find_single_rel_for_clauses(root, clauses); + + /* + * If there's exactly one clause that references a single relation, just + * go directly to clause_selectivity_ext(). None of what we might do below + * is relevant. If multiple relations are referenced, we must NOT + * short-circuit because we need to detect join stats opportunities. + */ + if (list_length(clauses) == 1 && rel != NULL) return clause_selectivity_ext(root, (Node *) linitial(clauses), varRelid, jointype, sjinfo, use_extended_stats); /* - * Determine if these clauses reference a single relation. If so, and if - * it has extended statistics, try to apply those. + * Try applying extended statistics for joins/parameterized scans. + * + * Note: FK-based joins are already handled by + * get_foreign_key_join_selectivity() which runs before + * clauselist_selectivity(). This primarily benefits: + * - Parameterized path estimation (before FK selectivity runs) + * - Non-FK join path estimation + */ + if (use_extended_stats && !rel) + { + /* + * Estimate as many clauses as possible using extended statistics. + * + * 'estimatedclauses' is populated with the 0-based list position + * index of clauses estimated here, and that should be ignored below. + */ + s1 *= statext_join_mcv_clauselist_selectivity(root, clauses, varRelid, + &estimatedclauses); + } + + /* + * Try applying extended statistics for single table scans. */ - rel = find_single_rel_for_clauses(root, clauses); if (use_extended_stats && rel && rel->rtekind == RTE_RELATION && rel->statlist != NIL) { /* @@ -150,9 +176,9 @@ clauselist_selectivity_ext(PlannerInfo *root, * 'estimatedclauses' is populated with the 0-based list position * index of clauses estimated here, and that should be ignored below. */ - s1 = statext_clauselist_selectivity(root, clauses, varRelid, - jointype, sjinfo, rel, - &estimatedclauses, false); + s1 *= statext_clauselist_selectivity(root, clauses, varRelid, + jointype, sjinfo, rel, + &estimatedclauses, false); } /* diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c index 1c575e56ff6..95c78116b38 100644 --- a/src/backend/optimizer/path/costsize.c +++ b/src/backend/optimizer/path/costsize.c @@ -109,6 +109,7 @@ #include "utils/selfuncs.h" #include "utils/spccache.h" #include "utils/tuplesort.h" +#include "statistics/statistics.h" #define LOG2(x) (log(x) / 0.693147180559945) @@ -5810,6 +5811,7 @@ get_foreign_key_join_selectivity(PlannerInfo *root, bool ref_is_outer; List *removedlist; ListCell *cell; + RelOptInfo *ref_rel; /* * This FK is not relevant unless it connects a baserel on one side of @@ -5937,7 +5939,8 @@ get_foreign_key_join_selectivity(PlannerInfo *root, /* * Finally we get to the payoff: estimate selectivity using the * knowledge that each referencing row will match exactly one row in - * the referenced table. + * the referenced table. If join MCV statistics exist for this FK, + * prefer the MCV-based selectivity over the default 1/ref_tuples. * * XXX that's not true in the presence of nulls in the referencing * column(s), so in principle we should derate the estimate for those. @@ -5957,6 +5960,8 @@ get_foreign_key_join_selectivity(PlannerInfo *root, * work, it is uncommon in practice to have an FK referencing a parent * table. So, at least for now, disregard inheritance here. */ + ref_rel = find_base_rel(root, fkinfo->ref_relid); + if (jointype == JOIN_SEMI || jointype == JOIN_ANTI) { /* @@ -5970,7 +5975,6 @@ get_foreign_key_join_selectivity(PlannerInfo *root, * restriction clauses, which is rows / tuples; but we must guard * against tuples == 0. */ - RelOptInfo *ref_rel = find_base_rel(root, fkinfo->ref_relid); double ref_tuples = Max(ref_rel->tuples, 1.0); fkselec *= ref_rel->rows / ref_tuples; @@ -5978,14 +5982,28 @@ get_foreign_key_join_selectivity(PlannerInfo *root, else { /* - * Otherwise, selectivity is exactly 1/referenced-table-size; but - * guard against tuples == 0. Note we should use the raw table - * tuple count, not any estimate of its filtered or joined size. + * Try the removed FK clause for join MCV stats. If a stat is + * found for the baserel pair, use its selectivity; otherwise + * fall back to the default FK selectivity + * (1/referenced-table-size) applied once per FK constraint. + * + * XXX Currently join stats support only single-condition joins, + * so skip the MCV lookup for multi-column FKs. */ - RelOptInfo *ref_rel = find_base_rel(root, fkinfo->ref_relid); double ref_tuples = Max(ref_rel->tuples, 1.0); + Selectivity mcv_sel = 0.0; + + if (list_length(removedlist) == 1) + { + RestrictInfo *fk_rinfo = (RestrictInfo *) linitial(removedlist); - fkselec *= 1.0 / ref_tuples; + mcv_sel = join_mcv_clause_selectivity(root, fk_rinfo); + } + + if (mcv_sel > 0) + fkselec *= mcv_sel; + else + fkselec *= 1.0 / ref_tuples; } /* diff --git a/src/backend/optimizer/util/plancat.c b/src/backend/optimizer/util/plancat.c index 7c4be174869..3050f00be29 100644 --- a/src/backend/optimizer/util/plancat.c +++ b/src/backend/optimizer/util/plancat.c @@ -1654,7 +1654,9 @@ get_relation_constraints(PlannerInfo *root, static void get_relation_statistics_worker(List **stainfos, RelOptInfo *rel, Oid statOid, bool inh, - Bitmapset *keys, List *exprs) + Bitmapset *keys, List *exprs, + List *joinrels, List *keyattrs, + List *keyrefs, List *joinconds) { Form_pg_statistic_ext_data dataForm; HeapTuple dtup; @@ -1705,6 +1707,10 @@ get_relation_statistics_worker(List **stainfos, RelOptInfo *rel, info->kind = STATS_EXT_MCV; info->keys = bms_copy(keys); info->exprs = exprs; + info->joinrels = joinrels; + info->keyattrs = keyattrs; + info->keyrefs = keyrefs; + info->joinconds = joinconds; *stainfos = lappend(*stainfos, info); } @@ -1754,6 +1760,12 @@ get_relation_statistics(PlannerInfo *root, RelOptInfo *rel, List *exprs = NIL; int i; + /* Join statistics fields */ + List *joinrels_list = NIL; + List *keyattrs_list = NIL; + List *keyrefs_list = NIL; + List *joinconds = NIL; + htup = SearchSysCache1(STATEXTOID, ObjectIdGetDatum(statOid)); if (!HeapTupleIsValid(htup)) elog(ERROR, "cache lookup failed for statistics object %u", statOid); @@ -1822,11 +1834,60 @@ get_relation_statistics(PlannerInfo *root, RelOptInfo *rel, } } + /* + * Get join statistics fields if present. Join stats have a non-null + * stxjoinrels array listing the other participating relations. + */ + { + bool isnull; + Datum datum; + + datum = SysCacheGetAttr(STATEXTOID, htup, + Anum_pg_statistic_ext_stxjoinrels, &isnull); + if (!isnull) + { + oidvector *jrels = (oidvector *) DatumGetPointer(datum); + int2vector *keyrefs_vec; + int nstxkeys; + char *condstr; + + for (i = 0; i < jrels->dim1; i++) + joinrels_list = lappend_oid(joinrels_list, + jrels->values[i]); + + /* stxkeys and stxkeyrefs */ + nstxkeys = staForm->stxkeys.dim1; + datum = SysCacheGetAttrNotNull(STATEXTOID, htup, + Anum_pg_statistic_ext_stxkeyrefs); + keyrefs_vec = (int2vector *) DatumGetPointer(datum); + for (i = 0; i < nstxkeys; i++) + { + keyattrs_list = lappend_int(keyattrs_list, + staForm->stxkeys.values[i]); + keyrefs_list = lappend_int(keyrefs_list, + keyrefs_vec->values[i]); + } + + /* stxjoinconds */ + datum = SysCacheGetAttrNotNull(STATEXTOID, htup, + Anum_pg_statistic_ext_stxjoinconds); + condstr = TextDatumGetCString(datum); + joinconds = (List *) stringToNode(condstr); + pfree(condstr); + } + } + /* extract statistics for possible values of stxdinherit flag */ - get_relation_statistics_worker(&stainfos, rel, statOid, true, keys, exprs); + get_relation_statistics_worker(&stainfos, rel, statOid, true, + keys, exprs, + joinrels_list, keyattrs_list, + keyrefs_list, joinconds); - get_relation_statistics_worker(&stainfos, rel, statOid, false, keys, exprs); + get_relation_statistics_worker(&stainfos, rel, statOid, false, + keys, exprs, + joinrels_list, keyattrs_list, + keyrefs_list, joinconds); ReleaseSysCache(htup); bms_free(keys); diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index ff4e1388c55..9989ebfd94b 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -4924,6 +4924,13 @@ stats_param: ColId $$->name = $1; $$->expr = NULL; } + | ColId indirection + { + /* Table-qualified column reference (e.g., tbl.col) */ + $$ = makeNode(StatsElem); + $$->name = NULL; + $$->expr = (Node *) makeColumnRef($1, $2, @1, yyscanner); + } | func_expr_windowless { $$ = makeNode(StatsElem); diff --git a/src/backend/parser/parse_utilcmd.c b/src/backend/parser/parse_utilcmd.c index 37071502a9f..3289663c25d 100644 --- a/src/backend/parser/parse_utilcmd.c +++ b/src/backend/parser/parse_utilcmd.c @@ -3142,12 +3142,53 @@ transformIndexStmt(Oid relid, IndexStmt *stmt, const char *queryString) return stmt; } +/* + * flatten_join_tree - recursively flatten a JoinExpr tree + * + * Opens each RangeVar leaf, adds it to the parser namespace, and appends + * the opened Relation to *rels. Appends each JoinExpr node to *joins. + * Both lists are in left-to-right tree order. + */ +static void +flatten_join_tree(ParseState *pstate, Node *node, + List **rels, List **joins) +{ + if (IsA(node, RangeVar)) + { + RangeVar *rv = (RangeVar *) node; + Relation rel; + ParseNamespaceItem *nsitem; + + rel = table_openrv(rv, ShareUpdateExclusiveLock); + nsitem = addRangeTableEntryForRelation(pstate, rel, + AccessShareLock, + rv->alias, false, true); + addNSItemToQuery(pstate, nsitem, false, true, true); + *rels = lappend(*rels, rel); + } + else if (IsA(node, JoinExpr)) + { + JoinExpr *j = (JoinExpr *) node; + + flatten_join_tree(pstate, j->larg, rels, joins); + flatten_join_tree(pstate, j->rarg, rels, joins); + *joins = lappend(*joins, j); + } + else + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("unsupported FROM clause element in join statistics"))); +} + /* * transformStatsStmt - parse analysis for CREATE STATISTICS * - * To avoid race conditions, it's important that this function relies only on - * the passed-in relid (and not on stmt->relation) to determine the target - * relation. + * For single-relation statistics, to avoid race conditions, it's important + * that this function relies only on the passed-in relid (and not on + * stmt->relation) to determine the target relation. + * + * For join statistics, relid is InvalidOid and tables are resolved directly + * from stmt->relations. */ CreateStatsStmt * transformStatsStmt(Oid relid, CreateStatsStmt *stmt, const char *queryString) @@ -3155,7 +3196,8 @@ transformStatsStmt(Oid relid, CreateStatsStmt *stmt, const char *queryString) ParseState *pstate; ParseNamespaceItem *nsitem; ListCell *l; - Relation rel; + Relation rel = NULL; + bool isjoin = !OidIsValid(relid); /* Nothing to do if statement already transformed. */ if (stmt->transformed) @@ -3165,18 +3207,122 @@ transformStatsStmt(Oid relid, CreateStatsStmt *stmt, const char *queryString) pstate = make_parsestate(NULL); pstate->p_sourcetext = queryString; - /* - * Put the parent table into the rtable so that the expressions can refer - * to its fields without qualification. Caller is responsible for locking - * relation, but we still need to open it. - */ - rel = relation_open(relid, NoLock); - nsitem = addRangeTableEntryForRelation(pstate, rel, - AccessShareLock, - NULL, false, true); + if (isjoin) + { + Node *fromNode = (Node *) linitial(stmt->relations); + List *rels = NIL; + List *joins = NIL; + int nrels; - /* no to join list, yes to namespaces */ - addNSItemToQuery(pstate, nsitem, false, true, true); + Assert(IsA(fromNode, JoinExpr)); + + /* Collect tables and join nodes from the JoinExpr node */ + flatten_join_tree(pstate, fromNode, &rels, &joins); + + nrels = list_length(rels); + Assert(nrels >= 2); + + /* + * Transform the join quals in each JoinExpr now that all tables are + * in the namespace, and collect them into stmt->stxjoinconds. + */ + foreach(l, joins) + { + JoinExpr *j = (JoinExpr *) lfirst(l); + + if (j->jointype != JOIN_INNER) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("join statistics are only supported for inner joins"))); + + if (j->quals) + { + j->quals = transformExpr(pstate, j->quals, + EXPR_KIND_JOIN_ON); + assign_expr_collations(pstate, j->quals); + stmt->stxjoinconds = lappend(stmt->stxjoinconds, j->quals); + } + } + + /* + * Validate join conditions. Each qual must be a simple OpExpr with + * Var operands using a mergejoinable equality operator. + */ + foreach(l, stmt->stxjoinconds) + { + OpExpr *opexpr; + Var *lvar; + Var *rvar; + + if (!IsA(lfirst(l), OpExpr)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("join statistics require a single equijoin condition per pair of tables"))); + + opexpr = (OpExpr *) lfirst(l); + + if (list_length(opexpr->args) != 2 || + !IsA(linitial(opexpr->args), Var) || + !IsA(lsecond(opexpr->args), Var)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("join statistics require simple equijoin conditions"))); + + lvar = (Var *) linitial(opexpr->args); + rvar = (Var *) lsecond(opexpr->args); + + if (lvar->varno < 1 || lvar->varno > nrels || + rvar->varno < 1 || rvar->varno > nrels) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("join condition references invalid relation"))); + + if (get_mergejoin_opfamilies(opexpr->opno) == NIL) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("join statistics require equijoin conditions"))); + } + + /* Reject join stats without any join conditions (e.g. CROSS JOIN) */ + if (stmt->stxjoinconds == NIL) + ereport(ERROR, + (errcode(ERRCODE_INVALID_OBJECT_DEFINITION), + errmsg("join statistics require at least one join condition"), + errhint("Use JOIN ... ON instead of CROSS JOIN."))); + + /* + * Extract resolved table OIDs. The anchor (stxrelid) is the first + * table in the FROM clause; the rest go to stxjoinrels. + */ + stmt->stxrelid = RelationGetRelid((Relation) linitial(rels)); + foreach(l, rels) + { + Relation r = (Relation) lfirst(l); + + if (foreach_current_index(l) > 0) + stmt->stxjoinrels = lappend_oid(stmt->stxjoinrels, + RelationGetRelid(r)); + table_close(r, NoLock); + } + + list_free(rels); + list_free(joins); + } + else + { + /* + * Put the parent table into the rtable so that the expressions can + * refer to its fields without qualification. Caller is responsible + * for locking relation, but we still need to open it. + */ + rel = relation_open(relid, NoLock); + nsitem = addRangeTableEntryForRelation(pstate, rel, + AccessShareLock, + NULL, false, true); + + /* no to join list, yes to namespaces */ + addNSItemToQuery(pstate, nsitem, false, true, true); + } /* take care of any expressions */ foreach(l, stmt->exprs) @@ -3194,19 +3340,32 @@ transformStatsStmt(Oid relid, CreateStatsStmt *stmt, const char *queryString) } } - /* - * Check that only the base rel is mentioned. (This should be dead code - * now that add_missing_from is history.) - */ - if (list_length(pstate->p_rtable) != 1) - ereport(ERROR, - (errcode(ERRCODE_INVALID_COLUMN_REFERENCE), - errmsg("statistics expressions can refer only to the table being referenced"))); + if (!isjoin) + { + /* + * Check that only the base rel is mentioned. (This should be dead + * code now that add_missing_from is history.) + */ + if (list_length(pstate->p_rtable) != 1) + ereport(ERROR, + (errcode(ERRCODE_INVALID_COLUMN_REFERENCE), + errmsg("statistics expressions can refer only to the table being referenced"))); - free_parsestate(pstate); + /* Close relation */ + table_close(rel, NoLock); + } + else + { + /* + * For extended join stats, we need at least 2 tables in the rtable. + */ + if (list_length(pstate->p_rtable) < 2) + ereport(ERROR, + (errcode(ERRCODE_INVALID_COLUMN_REFERENCE), + errmsg("extended join statistics must reference at least two tables"))); + } - /* Close relation */ - table_close(rel, NoLock); + free_parsestate(pstate); /* Mark statement as successfully transformed */ stmt->transformed = true; diff --git a/src/backend/statistics/Makefile b/src/backend/statistics/Makefile index 7ff5938b027..9639e0b95f5 100644 --- a/src/backend/statistics/Makefile +++ b/src/backend/statistics/Makefile @@ -17,6 +17,7 @@ OBJS = \ dependencies.o \ extended_stats.o \ extended_stats_funcs.o \ + join_mcv.o \ mcv.o \ mvdistinct.o \ relation_stats.o \ diff --git a/src/backend/statistics/extended_stats.c b/src/backend/statistics/extended_stats.c index 2b83355d26e..74ac6fd43fe 100644 --- a/src/backend/statistics/extended_stats.c +++ b/src/backend/statistics/extended_stats.c @@ -71,6 +71,20 @@ typedef struct StatExtEntry List *types; /* 'char' list of enabled statistics kinds */ int stattarget; /* statistics target (-1 for default) */ List *exprs; /* expressions */ + + /* + * Join statistics fields (NULL/invalid for single-table stats). + * + * For joins, the columns Bitmapset is insufficient because the same + * attnum can appear from different relations. These arrays preserve the + * full per-column mapping. + */ + int16 *attnums; /* attribute numbers, one per stats column */ + int nattnums; /* length of attnums[] and attref_varnos[] */ + int16 *attref_varnos; /* source relation varno for each attnum */ + Oid *joinrels; /* other relation OIDs */ + int njoinrels; /* number of other relations */ + List *joinconds; /* List of OpExpr: join conditions */ } StatExtEntry; @@ -158,6 +172,7 @@ BuildRelationExtStatistics(Relation onerel, bool inh, double totalrows, MCVList *mcv = NULL; Datum exprstats = (Datum) 0; VacAttrStats **stats; + VacAttrStats **mcv_stats = NULL; ListCell *lc2; int stattarget; StatsBuildData *data; @@ -165,10 +180,12 @@ BuildRelationExtStatistics(Relation onerel, bool inh, double totalrows, /* * Check if we can build these stats based on the column analyzed. If * not, report this fact (except in autovacuum) and move on. + * + * For join stats this may return NULL; they are built separately. */ stats = lookup_var_attr_stats(stat->columns, stat->exprs, natts, vacattrstats); - if (!stats) + if (!stats && stat->njoinrels == 0) { if (!AmAutoVacuumWorkerProcess()) ereport(WARNING, @@ -181,21 +198,40 @@ BuildRelationExtStatistics(Relation onerel, bool inh, double totalrows, continue; } - /* compute statistics target for this statistics object */ - stattarget = statext_compute_stattarget(stat->stattarget, - bms_num_members(stat->columns), - stats); + /* Join stats are built separately later */ + if (stat->njoinrels > 0) + { + stattarget = stat->stattarget >= 0 ? stat->stattarget + : default_statistics_target; - /* - * Don't rebuild statistics objects with statistics target set to 0 - * (we just leave the existing values around, just like we do for - * regular per-column statistics). - */ - if (stattarget == 0) - continue; + /* + * Don't rebuild statistics objects with statistics target set to + * 0 (we just leave the existing values around, just like we do + * for regular per-column statistics). + */ + if (stattarget == 0) + continue; + + data = NULL; + } + else + { + /* compute statistics target for this statistics object */ + stattarget = statext_compute_stattarget(stat->stattarget, + bms_num_members(stat->columns), + stats); - /* evaluate expressions (if the statistics object has any) */ - data = make_build_data(onerel, stat, numrows, rows, stats, stattarget); + /* + * Don't rebuild statistics objects with statistics target set to + * 0 (we just leave the existing values around, just like we do + * for regular per-column statistics). + */ + if (stattarget == 0) + continue; + + /* evaluate expressions (if the statistics object has any) */ + data = make_build_data(onerel, stat, numrows, rows, stats, stattarget); + } /* compute statistic of each requested type */ foreach(lc2, stat->types) @@ -207,7 +243,78 @@ BuildRelationExtStatistics(Relation onerel, bool inh, double totalrows, else if (t == STATS_EXT_DEPENDENCIES) dependencies = statext_dependencies_build(data); else if (t == STATS_EXT_MCV) - mcv = statext_mcv_build(data, totalrows, stattarget); + { + if (stat->njoinrels > 0) + { + VacAttrStats **join_mcv_stats = NULL; + int16 jleft[INDEX_MAX_KEYS]; + int16 jright[INDEX_MAX_KEYS]; + Oid jops[INDEX_MAX_KEYS]; + int njq = 0; + ListCell *jlc; + + /* Warn and skip n-way joins (not yet supported) */ + if (stat->njoinrels > 1) + { + ereport(WARNING, + (errmsg("join statistics on more than two tables are not yet supported, skipping \"%s\"", + stat->name))); + break; + } + + /* + * Extract flat arrays from joinconds for the build + * function. For n-way (njoinrels > 1) the build function + * will warn and skip. + */ + foreach(jlc, stat->joinconds) + { + OpExpr *op = (OpExpr *) lfirst(jlc); + Var *lv = (Var *) linitial(op->args); + Var *rv = (Var *) lsecond(op->args); + + if (njq >= INDEX_MAX_KEYS) + break; + + /* + * Normalize: jleft = sampling side (lower varno), + * jright = indexed/probed side (higher varno). + */ + if (lv->varno < rv->varno) + { + jleft[njq] = lv->varattno; + jright[njq] = rv->varattno; + } + else + { + jleft[njq] = rv->varattno; + jright[njq] = lv->varattno; + } + jops[njq] = op->opno; + njq++; + } + + mcv = statext_join_mcv_build( + onerel, + stat->joinrels, + stat->njoinrels, + jleft, jright, + jops, njq, + stat->attnums, + stat->attref_varnos, + stat->nattnums, + stattarget, + numrows, rows, totalrows, + &join_mcv_stats); + if (join_mcv_stats) + mcv_stats = join_mcv_stats; + } + else + { + mcv = statext_mcv_build(data, totalrows, stattarget); + mcv_stats = stats; + } + } else if (t == STATS_EXT_EXPRESSIONS) { AnlExprData *exprdata; @@ -228,7 +335,7 @@ BuildRelationExtStatistics(Relation onerel, bool inh, double totalrows, /* store the statistics in the catalog */ statext_store(stat->statOid, inh, - ndistinct, dependencies, mcv, exprstats, stats); + ndistinct, dependencies, mcv, exprstats, mcv_stats); /* for reporting progress */ pgstat_progress_update_param(PROGRESS_ANALYZE_EXT_STATS_COMPUTED, @@ -546,6 +653,55 @@ fetch_statentries_for_relation(Relation pg_statext, Relation rel) entry->exprs = exprs; + /* + * Fetch join statistics fields. These are all NULL for single-table + * statistics. + */ + datum = SysCacheGetAttr(STATEXTOID, htup, + Anum_pg_statistic_ext_stxjoinrels, &isnull); + if (!isnull) + { + oidvector *ov = (oidvector *) DatumGetPointer(datum); + + entry->njoinrels = ov->dim1; + entry->joinrels = (Oid *) palloc(ov->dim1 * sizeof(Oid)); + memcpy(entry->joinrels, ov->values, ov->dim1 * sizeof(Oid)); + + entry->nattnums = staForm->stxkeys.dim1; + entry->attnums = palloc(entry->nattnums * sizeof(int16)); + for (i = 0; i < entry->nattnums; i++) + entry->attnums[i] = staForm->stxkeys.values[i]; + } + else + { + entry->njoinrels = 0; + entry->joinrels = NULL; + } + + datum = SysCacheGetAttr(STATEXTOID, htup, + Anum_pg_statistic_ext_stxkeyrefs, &isnull); + if (!isnull) + { + int2vector *iv = (int2vector *) DatumGetPointer(datum); + + entry->attref_varnos = (int16 *) palloc(iv->dim1 * sizeof(int16)); + memcpy(entry->attref_varnos, iv->values, iv->dim1 * sizeof(int16)); + } + else + entry->attref_varnos = NULL; + + datum = SysCacheGetAttr(STATEXTOID, htup, + Anum_pg_statistic_ext_stxjoinconds, &isnull); + if (!isnull) + { + char *str = TextDatumGetCString(datum); + + entry->joinconds = (List *) stringToNode(str); + pfree(str); + } + else + entry->joinconds = NIL; + result = lappend(result, entry); } @@ -2022,6 +2178,67 @@ statext_mcv_clauselist_selectivity(PlannerInfo *root, List *clauses, int varReli return sel; } +/* + * statext_join_mcv_clauselist_selectivity + * Estimate join clause selectivity using join MCV statistics. + * + * Iterates over the clause list and, for each join clause (Var = Var across + * different relations), extracts the baserel pair from the Vars and checks + * whether a join MCV stat for that pair can provide a better selectivity + * estimate. This per baserel pair approach produces the same result + * regardless of which component rel pair is used to form the joinrel, + * because it goes directly to the baserels' statlists via Var.varno + * rather than inspecting the join tree structure. + * + * The selectivity for each baserel pair is adjusted to account for filter + * correlations: the raw MCV selectivity (P(join AND filter)) is divided by + * the filter selectivity (already reflected in base rel rows) to yield + * P(join | filter). + * + * Returns the product of estimated selectivities (1.0 for clauses without + * applicable stats). 'estimatedclauses' is populated with the 0-based list + * position indexes of clauses estimated here. + */ +Selectivity +statext_join_mcv_clauselist_selectivity(PlannerInfo *root, + List *clauses, + int varRelid, + Bitmapset **estimatedclauses) +{ + Selectivity result = 1.0; + int clause_idx = -1; + ListCell *lc; + + foreach(lc, clauses) + { + RestrictInfo *rinfo; + Selectivity selec; + + clause_idx++; + + if (!IsA(lfirst(lc), RestrictInfo)) + continue; + + rinfo = (RestrictInfo *) lfirst(lc); + + /* + * Skip clauses already estimated (e.g., filter clauses from a prior + * join stat) + */ + if (bms_is_member(clause_idx, *estimatedclauses)) + continue; + + selec = join_mcv_clause_selectivity(root, rinfo); + if (selec <= 0) + continue; + + result *= selec; + *estimatedclauses = bms_add_member(*estimatedclauses, clause_idx); + } + + return result; +} + /* * statext_clauselist_selectivity * Estimate clauses using the best multi-column statistics. diff --git a/src/backend/statistics/join_mcv.c b/src/backend/statistics/join_mcv.c new file mode 100644 index 00000000000..7a0eedb1c5d --- /dev/null +++ b/src/backend/statistics/join_mcv.c @@ -0,0 +1,1723 @@ +/*------------------------------------------------------------------------- + * + * join_mcv.c + * POSTGRES join MCV lists + * + * Join MCV statistics capture correlations between columns across an + * equijoin. The "anchor" relation owns the statistics object (stxrelid) + * and is sampled during ANALYZE; the "other" relation (stxjoinrels) is + * probed via index lookup to build a weighted sample of the join result. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/statistics/join_mcv.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/amapi.h" +#include "access/genam.h" +#include "access/heapam.h" +#include "access/htup_details.h" +#include "access/table.h" +#include "access/tableam.h" +#include "catalog/pg_index.h" +#include "catalog/pg_statistic_ext.h" +#include "common/pg_prng.h" +#include "executor/tuptable.h" +#include "fmgr.h" +#include "optimizer/optimizer.h" +#include "optimizer/pathnode.h" +#include "optimizer/paths.h" +#include "statistics/extended_stats_internal.h" +#include "statistics/statistics.h" +#include "utils/array.h" +#include "utils/datum.h" +#include "utils/lsyscache.h" +#include "utils/relcache.h" +#include "utils/selfuncs.h" +#include "utils/sampling.h" +#include "utils/snapmgr.h" +#include "utils/syscache.h" +#include "utils/typcache.h" + + + +/* + * find_index_for_operator + * Find an index that supports lookup on the given column using + * the specified operator. + * + * Checks that the operator belongs to the index's opfamily for the + * target column. Works for btree, hash, and any other AM that supports + * amgettuple and registers the operator in its opfamily. Partial indexes + * are excluded. + * + * Returns the index OID, or InvalidOid if no suitable index found. + * Prefers unique indexes, then fewer columns. + */ +Oid +find_index_for_operator(Relation rel, AttrNumber attnum, Oid eq_op) +{ + List *indexlist; + ListCell *lc; + Oid best_idx = InvalidOid; + bool best_is_unique = false; + int best_nattnums = INT_MAX; + + indexlist = RelationGetIndexList(rel); + + foreach(lc, indexlist) + { + Oid indexoid = lfirst_oid(lc); + Relation idxrel; + Form_pg_index idxform; + Oid opfamily; + + idxrel = index_open(indexoid, AccessShareLock); + idxform = idxrel->rd_index; + + /* + * The index must be valid, ready, live, and non-partial (partial + * indexes only contain rows matching their predicate, which would + * bias the sample). The target column must be the leading key column + * so the AM can look up by it (a multi-column index is fine). The AM + * must support amgettuple for single-tuple scans (this excludes BRIN, + * which only supports bitmap scans). The index's opfamily must + * contain our equality operator. + */ + if (idxform->indisvalid && idxform->indisready && + idxform->indislive && + heap_attisnull(idxrel->rd_indextuple, Anum_pg_index_indpred, NULL) && + idxform->indnkeyatts >= 1 && + idxform->indkey.values[0] == attnum && + idxrel->rd_indam->amgettuple != NULL) + { + opfamily = idxrel->rd_opfamily[0]; + if (op_in_opfamily(eq_op, opfamily)) + { + if (!OidIsValid(best_idx) || + (!best_is_unique && idxform->indisunique) || + (idxform->indisunique == best_is_unique && + idxform->indnkeyatts < best_nattnums)) + { + best_idx = indexoid; + best_is_unique = idxform->indisunique; + best_nattnums = idxform->indnkeyatts; + } + } + } + + index_close(idxrel, AccessShareLock); + } + + list_free(indexlist); + return best_idx; +} + +/* + * sample_index -- Build a weighted sample of the join result using an index. + * + * Implements Algorithm 1 from Leis et al., CIDR 2017, Section 3.1. Given + * sample tuples S from the anchor table and an index I on the other table's + * join column, this produces a representative sample of the join result. + * + * For each tuple in S, we probe the index to count matching tuples, producing + * a "count per tuple" (cpt) array and the total match count (sum). + * + * We then sample min(sum, n) positions from [0, sum), where n is the maximum + * sample size requested by the caller. The cpt counts define contiguous + * intervals in [0, sum); each sampled position falls into exactly one + * interval, identifying both the cpt entry and the offset within it. + * + * For each sampled position, we re-probe the index for the corresponding + * sample tuple and advance to the target offset to fetch the matching row. + * The requested columns are extracted from each fetched tuple. + * + * When sum <= n every match is collected without sampling. + * + * Returns a StatsBuildData with extracted column values from the matched + * side, or NULL if no matches found. *result_sample_indices is set to an + * array mapping each output row to its input tuple index in sample_tuples, + * so the caller can extract columns from the anchor side as well. + */ +static StatsBuildData * +sample_index(HeapTuple *sample_tuples, int nsample, TupleDesc sample_desc, + AttrNumber joinkey_attnum, + Relation heap_rel, Relation idx_rel, + int idx_strategy, + int n, + AttrNumber *extract_attnums, int16 *extract_typlens, + bool *extract_typbyvals, int nextract, + int **result_sample_indices) +{ + IndexScanDesc iscan; + ScanKeyData skey[1]; + TupleTableSlot *slot; + Snapshot snapshot; + bool pushed_snapshot = false; + int i; + int sid_idx; + int64 prefix_sum; + int nresults; + int *sample_indices; /* maps each output row to its input index */ + + /* cpt ("count per tuple") = sequence of (tuple_index, count) pairs */ + int64 *cpt_count; + int *cpt_tupleidx; + int cpt_len; + int64 sum; + + /* sid = sampled positions (Algorithm S variables) */ + int64 *sid; + int sid_len; + int64 t; /* population elements scanned */ + int m; /* elements selected so far */ + + /* result_data = output data */ + StatsBuildData *result_data; + + /* scan key derivation */ + Oid anchor_type; + Oid scan_eq_op; + RegProcedure eq_proc; + + /* Derive the index-perspective equality operator and comparison proc. */ + anchor_type = TupleDescAttr(sample_desc, joinkey_attnum - 1)->atttypid; + scan_eq_op = get_opfamily_member(idx_rel->rd_opfamily[0], + idx_rel->rd_opcintype[0], + anchor_type, + idx_strategy); + if (!OidIsValid(scan_eq_op)) + elog(ERROR, "could not find equality operator for types %u and %u in opfamily %u", + idx_rel->rd_opcintype[0], anchor_type, idx_rel->rd_opfamily[0]); + eq_proc = get_opcode(scan_eq_op); + + /* + * The index scan needs an active snapshot for tuple visibility. The + * caller (vacuum.c) always pushes one before we get here, but be + * defensive in case this is ever called from another path. + */ + if (!ActiveSnapshotSet()) + { + PushActiveSnapshot(GetTransactionSnapshot()); + pushed_snapshot = true; + } + snapshot = GetActiveSnapshot(); + + slot = MakeSingleTupleTableSlot(RelationGetDescr(heap_rel), + &TTSOpsBufferHeapTuple); + iscan = index_beginscan(heap_rel, idx_rel, snapshot, NULL, 1, 0, 0); + + /* Count matching tuples per sample tuple to build the cpt array. */ + cpt_count = palloc_array(int64, nsample); + cpt_tupleidx = palloc_array(int, nsample); + sum = 0; + cpt_len = 0; + + for (i = 0; i < nsample; i++) + { + Datum joinkey_value; + bool joinkey_isnull; + int64 count; + + joinkey_value = heap_getattr(sample_tuples[i], joinkey_attnum, sample_desc, &joinkey_isnull); + if (joinkey_isnull) + continue; + + /* I.lookup(t).count */ + ScanKeyEntryInitialize(&skey[0], 0, 1, idx_strategy, anchor_type, + idx_rel->rd_indcollation[0], + eq_proc, joinkey_value); + index_rescan(iscan, skey, 1, NULL, 0); + + count = 0; + while (index_getnext_slot(iscan, ForwardScanDirection, slot)) + { + /* skip if AM requires recheck and tuple doesn't actually match */ + if (iscan->xs_recheck) + { + Datum val; + bool isnull; + + val = slot_getattr(slot, + idx_rel->rd_index->indkey.values[0], + &isnull); + if (isnull || + !DatumGetBool(FunctionCall2Coll(&skey[0].sk_func, + skey[0].sk_collation, + val, + skey[0].sk_argument))) + continue; + } + count++; + } + + cpt_count[cpt_len] = count; + cpt_tupleidx[cpt_len] = i; + sum += count; + cpt_len++; + } + + if (sum == 0) + { + index_endscan(iscan); + ExecDropSingleTupleTableSlot(slot); + if (pushed_snapshot) + PopActiveSnapshot(); + pfree(cpt_count); + pfree(cpt_tupleidx); + return NULL; + } + + /* + * "sid = sample non-negative integers < sum, |sid| = min{sum, n}" + * + * When sum <= n, sid = {0, 1, ..., sum-1} -- every position is selected. + */ + sid_len = Min(sum, n); + + if (sum <= n) + { + sid = palloc_array(int64, sid_len); + for (i = 0; i < sid_len; i++) + sid[i] = i; + } + else + { + /* + * Draw positions without replacement using Knuth's Algorithm S (Knuth + * 3.4.2). This is the same algorithm used by BlockSampler for + * table-block sampling. The output is naturally sorted, so no + * separate qsort step is needed. + */ + sid = palloc_array(int64, sid_len); + t = 0; + m = 0; + + while (m < sid_len) + { + int64 K = sum - t; /* remaining positions */ + int k = sid_len - m; /* samples still needed */ + double p; /* probability to skip position */ + double V; /* random */ + + Assert(sum > t && sid_len > m); /* hence K > 0 and k > 0 */ + + if (k >= K) + { + /* need all the rest */ + sid[m++] = t++; + } + else + { + V = sampler_random_fract(&pg_global_prng_state); + p = 1.0 - (double) k / (double) K; + + while (V < p) + { + /* skip */ + t++; + K--; /* keep K == sum - t */ + + /* adjust p to be new cutoff point in reduced range */ + p *= 1.0 - (double) k / (double) K; + } + + /* select */ + sid[m++] = t++; + } + } + } + + sample_indices = palloc_array(int, sid_len); + result_data = palloc(sizeof(StatsBuildData)); + result_data->nattnums = nextract; + result_data->numrows = 0; + result_data->attnums = palloc_array(AttrNumber, nextract); + result_data->stats = palloc0_array(VacAttrStats *, nextract); + result_data->values = palloc_array(Datum *, nextract); + result_data->nulls = palloc_array(bool *, nextract); + + for (i = 0; i < nextract; i++) + { + result_data->attnums[i] = extract_attnums[i]; + result_data->values[i] = palloc_array(Datum, sid_len); + result_data->nulls[i] = palloc_array(bool, sid_len); + } + + /* + * Walk cpt and sid in parallel (both sorted by position) using a running + * prefix sum to identify the cpt entry and offset for each sampled + * position, then fetch the matching tuple from the index. + */ + nresults = 0; + sid_idx = 0; + prefix_sum = 0; + + for (i = 0; i < cpt_len && sid_idx < sid_len; i++) + { + /* this cpt entry owns positions [prefix_sum, next_prefix) */ + int64 next_prefix = prefix_sum + cpt_count[i]; + int64 offset; + Datum joinkey_value; + bool joinkey_isnull; + int64 j; + int f; + bool scan_started; + bool scan_exhausted; + + scan_started = false; + scan_exhausted = false; + j = -1; + + while (sid_idx < sid_len && sid[sid_idx] < next_prefix) + { + /* offset within this cpt entry's interval */ + offset = sid[sid_idx] - prefix_sum; + + /* + * If a previous advance within this cpt entry ran out of tuples, + * all later offsets (which are larger) will too. + */ + if (scan_exhausted) + { + sid_idx++; + continue; + } + + /* + * Start the index scan on the first sid for this cpt entry; + * subsequent sids reuse the open scan. + */ + if (!scan_started) + { + joinkey_value = heap_getattr(sample_tuples[cpt_tupleidx[i]], + joinkey_attnum, sample_desc, + &joinkey_isnull); + Assert(!joinkey_isnull); + + ScanKeyEntryInitialize(&skey[0], 0, 1, idx_strategy, + anchor_type, + idx_rel->rd_indcollation[0], + eq_proc, joinkey_value); + index_rescan(iscan, skey, 1, NULL, 0); + scan_started = true; + } + + /* Advance to the target offset */ + while (index_getnext_slot(iscan, ForwardScanDirection, slot)) + { + /* skip if AM requires recheck and tuple doesn't match */ + if (iscan->xs_recheck) + { + Datum val; + bool valisnull; + + val = slot_getattr(slot, + idx_rel->rd_index->indkey.values[0], + &valisnull); + if (valisnull || + !DatumGetBool(FunctionCall2Coll(&skey[0].sk_func, + skey[0].sk_collation, + val, + skey[0].sk_argument))) + continue; + } + j++; + if (j >= offset) + break; + } + + if (j >= offset) + { + /* Extract requested columns from tA */ + for (f = 0; f < nextract; f++) + { + result_data->values[f][nresults] = + slot_getattr(slot, extract_attnums[f], + &result_data->nulls[f][nresults]); + + if (!result_data->nulls[f][nresults] && !extract_typbyvals[f]) + result_data->values[f][nresults] = + datumCopy(result_data->values[f][nresults], + extract_typbyvals[f], + extract_typlens[f]); + } + + /* Record which input tuple this output row came from */ + sample_indices[nresults] = cpt_tupleidx[i]; + nresults++; + } + else + { + /* Scan exhausted; skip remaining sids for this entry */ + scan_exhausted = true; + } + + sid_idx++; + } + + prefix_sum = next_prefix; + } + + result_data->numrows = nresults; + + pfree(sid); + pfree(cpt_count); + pfree(cpt_tupleidx); + index_endscan(iscan); + ExecDropSingleTupleTableSlot(slot); + + if (pushed_snapshot) + PopActiveSnapshot(); + + if (nresults == 0) + { + pfree(sample_indices); + return NULL; + } + + *result_sample_indices = sample_indices; + return result_data; +} + +/* + * statext_join_mcv_build + * Build join MCV statistics using index-based join sampling. + * + * Samples the join between the anchor table and the other table by scanning + * anchor rows and probing via index lookup, then feeds the merged sample + * into statext_mcv_build() to produce a standard MCVList. + * + * Currently only 2-way joins are supported; warns and returns NULL for + * n-way (njoinrels > 1). + */ +MCVList * +statext_join_mcv_build(Relation anchor_rel, + Oid *joinrel_oids, + int njoinrels, + int16 *joinleft, + int16 *joinright, + Oid *joinops, + int njoinquals, + int16 *stxkeys, + int16 *keyrefs, + int nkeys, + int stattarget, + int numrows, HeapTuple *rows, + double totalrows, + VacAttrStats ***result_stats) +{ + Relation other_rel; + Oid idx_oid; + Relation idx_rel; + TupleDesc other_desc; + TupleDesc anchor_desc; + int nfilters; + AttrNumber *other_attnums; + int16 *other_typlens; + bool *other_typbyvals; + int nother; + StatsBuildData *data; + StatsBuildData *merged; + int other_col; + MCVList *mcv; + Form_pg_attribute attr; + int i; + int n; + int *sample_indices; + + /* Currently only 2-way joins are supported for collection */ + if (njoinrels != 1) + return NULL; + + if (njoinquals < 1) + return NULL; + + /* Open the other table (joinrel_oids[0]) */ + other_rel = table_open(joinrel_oids[0], AccessShareLock); + other_desc = RelationGetDescr(other_rel); + + /* Find an index I that supports equality lookup with our operator */ + idx_oid = find_index_for_operator(other_rel, joinright[0], joinops[0]); + if (!OidIsValid(idx_oid)) + { + ereport(WARNING, + (errmsg("no suitable index on \"%s\" column \"%s\" for join statistics", + RelationGetRelationName(other_rel), + NameStr(TupleDescAttr(other_desc, joinright[0] - 1)->attname)))); + table_close(other_rel, AccessShareLock); + return NULL; + } + + idx_rel = index_open(idx_oid, AccessShareLock); + + /* + * Split stxkey columns into anchor-side and other-side. keyrefs[i] + * identifies which relation stxkeys[i] belongs to: 1 = anchor (stxrelid), + * 2 = first joinrel (stxjoinrels[0]), etc. sample_index can only extract + * from the probed table; anchor-side columns are extracted afterwards + * using sample_indices. + */ + nfilters = nkeys; + anchor_desc = RelationGetDescr(anchor_rel); + nother = 0; + + other_attnums = palloc_array(AttrNumber, nfilters); + other_typlens = palloc_array(int16, nfilters); + other_typbyvals = palloc_array(bool, nfilters); + + for (i = 0; i < nkeys; i++) + { + AttrNumber attnum = stxkeys[i]; + + if (keyrefs[i] == 1) + continue; /* anchor-side; extracted later */ + + /* Other-side column */ + attr = TupleDescAttr(other_desc, attnum - 1); + other_attnums[nother] = attnum; + other_typlens[nother] = attr->attlen; + other_typbyvals[nother] = attr->attbyval; + nother++; + } + + /* + * Run Algorithm 1: sample the join, extracting other-side columns. + * Request stattarget * 100 rows to give statext_mcv_build() enough data + * to identify the most common values reliably. + */ + n = stattarget * 100; + data = sample_index(rows, numrows, anchor_desc, + joinleft[0], + other_rel, idx_rel, + get_op_opfamily_strategy(joinops[0], + idx_rel->rd_opfamily[0]), + n, + other_attnums, other_typlens, + other_typbyvals, nother, + &sample_indices); + + index_close(idx_rel, AccessShareLock); + + if (data == NULL) + { + table_close(other_rel, AccessShareLock); + return NULL; + } + + /* + * Rebuild StatsBuildData in stxkeys order with virtual column numbers. + * + * The join result is a virtual relation. Each stxkeys position gets a + * virtual column number (1, 2, 3, ...). sample_index produced data for + * other-side columns; we extract anchor-side columns from the ANALYZE + * sample using sample_indices. The final data is ordered by stxkeys + * position, ensuring unique attnums regardless of base-table attnum + * collisions. + */ + other_col = 0; + merged = palloc(sizeof(StatsBuildData)); + merged->nattnums = nkeys; + merged->numrows = data->numrows; + merged->attnums = palloc_array(AttrNumber, nkeys); + merged->stats = palloc0_array(VacAttrStats *, nkeys); + merged->values = palloc_array(Datum *, nkeys); + merged->nulls = palloc_array(bool *, nkeys); + + for (i = 0; i < nkeys; i++) + { + AttrNumber base_attnum = stxkeys[i]; + VacAttrStats *s; + Form_pg_attribute attr; + TupleDesc col_desc; + HeapTuple typtuple; + int r; + + /* Virtual column number: 1-based position in stxkeys */ + merged->attnums[i] = i + 1; + + if (keyrefs[i] == 1) + { + /* Anchor-side: extract from rows[] via sample_indices */ + col_desc = anchor_desc; + attr = TupleDescAttr(anchor_desc, base_attnum - 1); + + merged->values[i] = palloc_array(Datum, data->numrows); + merged->nulls[i] = palloc_array(bool, data->numrows); + + for (r = 0; r < data->numrows; r++) + { + merged->values[i][r] = + heap_getattr(rows[sample_indices[r]], + base_attnum, + anchor_desc, + &merged->nulls[i][r]); + + if (!merged->nulls[i][r] && !attr->attbyval) + merged->values[i][r] = + datumCopy(merged->values[i][r], + attr->attbyval, attr->attlen); + } + } + else + { + /* Other-side: already extracted by sample_index */ + col_desc = other_desc; + attr = TupleDescAttr(other_desc, base_attnum - 1); + + merged->values[i] = data->values[other_col]; + merged->nulls[i] = data->nulls[other_col]; + other_col++; + } + + /* Build VacAttrStats with virtual attnum */ + s = palloc0(sizeof(VacAttrStats)); + s->tupDesc = col_desc; + s->tupattnum = i + 1; /* virtual column number */ + s->attrtypid = attr->atttypid; + s->attrtypmod = attr->atttypmod; + s->attrcollid = attr->attcollation; + + typtuple = SearchSysCache1(TYPEOID, + ObjectIdGetDatum(attr->atttypid)); + if (HeapTupleIsValid(typtuple)) + { + s->attrtype = (Form_pg_type) palloc(sizeof(FormData_pg_type)); + memcpy(s->attrtype, GETSTRUCT(typtuple), + sizeof(FormData_pg_type)); + ReleaseSysCache(typtuple); + } + merged->stats[i] = s; + } + + table_close(other_rel, AccessShareLock); + + if (sample_indices) + pfree(sample_indices); + pfree(other_attnums); + pfree(other_typlens); + pfree(other_typbyvals); + + if (result_stats) + *result_stats = merged->stats; + + mcv = statext_mcv_build(merged, totalrows, stattarget); + + return mcv; +} + +/* + * join_mcv_clauselist_selectivity - + * Apply join MCV statistics to estimate selectivity. + * + * Given a join MCVList and filter values, compute the selectivity by + * summing the frequencies of MCV items that match the filter criteria. + * stxkey_indexes contains 0-based stxkey positions, as produced by + * match_join_stat(). + * + * For single-column queries on multi-column stats, we compute the + * marginal distribution by summing frequencies across non-queried + * dimensions. + * + * For multi-column queries, we match ALL queried columns with their + * corresponding dimensions and find MCV items where all dimensions match. + */ +static Selectivity +join_mcv_clauselist_selectivity(MCVList *mcvlist, + List *filter_values, + List *stxkey_indexes, + List *filter_collations, + double ndistinct) +{ + int item_idx; + Selectivity total_sel; + Selectivity mcv_totalsel; + ListCell *lc; + int nkeys; + int i; + FmgrInfo *eq_funcs; + FunctionCallInfo *fcinfo_arr; + + if (!mcvlist || mcvlist->nitems == 0 || filter_values == NIL || stxkey_indexes == NIL) + return 0.0; + + nkeys = list_length(stxkey_indexes); + + if (nkeys > STATS_MAX_DIMENSIONS) + return 0.0; + + eq_funcs = palloc_array(FmgrInfo, nkeys); + fcinfo_arr = palloc_array(FunctionCallInfo, nkeys); + + for (i = 0; i < nkeys; i++) + { + int key_idx = list_nth_int(stxkey_indexes, i); + Oid typoid = mcvlist->types[key_idx]; + Oid coll = list_nth_oid(filter_collations, i); + TypeCacheEntry *typentry; + + typentry = lookup_type_cache(typoid, TYPECACHE_EQ_OPR_FINFO); + if (!OidIsValid(typentry->eq_opr)) + return 0.0; + + fmgr_info_copy(&eq_funcs[i], &typentry->eq_opr_finfo, + CurrentMemoryContext); + + fcinfo_arr[i] = palloc(SizeForFunctionCallInfo(2)); + InitFunctionCallInfoData(*fcinfo_arr[i], &eq_funcs[i], + 2, coll, NULL, NULL); + } + + total_sel = 0.0; + + mcv_totalsel = 0.0; + for (item_idx = 0; item_idx < mcvlist->nitems; item_idx++) + mcv_totalsel += mcvlist->items[item_idx].frequency; + + if (nkeys == 1) + { + int key_idx = list_nth_int(stxkey_indexes, 0); + bool *matched_items; + int nvalues = list_length(filter_values); + int nmatched = 0; + int mcv_nvalues; + + matched_items = palloc0_array(bool, mcvlist->nitems); + + /* + * Count distinct values of key_idx in the MCV. For single-dim MCVs + * every item has a unique value, so mcv_nvalues == nitems. For + * multi-dim MCVs multiple items can share the same value on key_idx + * (e.g., (A,red) and (B,red) share red), so we must scan to count. + * This is used in the non-MCV estimation guard and denominator, + * analogous to sslot.nvalues in eqjoinsel_inner. + */ + if (mcvlist->ndimensions == 1) + mcv_nvalues = mcvlist->nitems; + else + { + Datum *values; + int nvals = 0; + SortSupportData ssup; + TypeCacheEntry *typentry; + Oid typoid = mcvlist->types[key_idx]; + Oid coll = list_nth_oid(filter_collations, 0); + + /* Extract non-NULL values for key_idx */ + values = palloc_array(Datum, mcvlist->nitems); + + for (item_idx = 0; item_idx < mcvlist->nitems; item_idx++) + { + MCVItem *item = &mcvlist->items[item_idx]; + + if (item->isnull[key_idx]) + continue; + + values[nvals++] = item->values[key_idx]; + } + + if (nvals == 0) + mcv_nvalues = 0; + else + { + /* Sort and count distinct values */ + typentry = lookup_type_cache(typoid, TYPECACHE_LT_OPR); + + memset(&ssup, 0, sizeof(SortSupportData)); + ssup.ssup_cxt = CurrentMemoryContext; + ssup.ssup_collation = coll; + ssup.ssup_nulls_first = false; + + PrepareSortSupportFromOrderingOp(typentry->lt_opr, &ssup); + + qsort_interruptible(values, nvals, sizeof(Datum), + compare_scalars_simple, &ssup); + + mcv_nvalues = 1; + for (i = 1; i < nvals; i++) + { + if (compare_datums_simple(values[i - 1], values[i], + &ssup) != 0) + mcv_nvalues++; + } + } + + pfree(values); + } + + foreach(lc, filter_values) + { + Datum filter_value = PointerGetDatum(lfirst(lc)); + bool found = false; + + fcinfo_arr[0]->args[1].value = filter_value; + fcinfo_arr[0]->args[1].isnull = false; + + for (item_idx = 0; item_idx < mcvlist->nitems; item_idx++) + { + MCVItem *item; + Datum fresult; + + if (matched_items[item_idx]) + continue; + + item = &mcvlist->items[item_idx]; + + if (item->isnull[key_idx]) + continue; + + fcinfo_arr[0]->args[0].value = item->values[key_idx]; + fcinfo_arr[0]->args[0].isnull = false; + fcinfo_arr[0]->isnull = false; + + fresult = FunctionCallInvoke(fcinfo_arr[0]); + + if (!fcinfo_arr[0]->isnull && DatumGetBool(fresult)) + { + total_sel += item->frequency; + matched_items[item_idx] = true; + found = true; + + if (mcvlist->ndimensions == 1) + break; + } + } + + if (found) + nmatched++; + } + + /* + * Non-MCV estimation: estimate the contribution of filter values not + * found in the MCV list. + * + * The MCV only tracks the top-K most frequent value combinations. + * Filter values outside the MCV may still exist in the data but their + * frequencies are not tracked. We estimate each non-MCV value's + * frequency as the average frequency of non-MCV values: + * + * other_freq = (1 - mcv_totalsel) / (ndistinct - mcv_nvalues) + * + * where mcv_totalsel is the sum of all MCV item frequencies, + * ndistinct is the per-column distinct count for the filter column, + * and mcv_nvalues is the number of distinct values of the filter + * column represented in the MCV (analogous to sslot.nvalues in + * eqjoinsel_inner). + * + * Skip when ndistinct <= mcv_nvalues (the MCV covers all distinct + * values of this column, so non-MCV values truly don't exist) or when + * ndistinct is unavailable (0). + */ + if (nmatched < nvalues && ndistinct > mcv_nvalues) + { + int nunmatched = nvalues - nmatched; + double other_freq; + + other_freq = (1.0 - mcv_totalsel) / (ndistinct - mcv_nvalues); + CLAMP_PROBABILITY(other_freq); + + total_sel += nunmatched * other_freq; + CLAMP_PROBABILITY(total_sel); + } + } + else + { + /* Multi-column: exact match on all dimensions */ + Datum *query_values; + + if (list_length(filter_values) != nkeys) + return 0.0; + + query_values = palloc_array(Datum, nkeys); + i = 0; + foreach(lc, filter_values) + { + List *val_list = (List *) lfirst(lc); + + if (list_length(val_list) != 1) + return 0.0; + query_values[i] = PointerGetDatum(linitial(val_list)); + i++; + } + + for (item_idx = 0; item_idx < mcvlist->nitems; item_idx++) + { + MCVItem *item = &mcvlist->items[item_idx]; + bool all_match = true; + + for (i = 0; i < nkeys; i++) + { + int key_idx = list_nth_int(stxkey_indexes, i); + Datum fresult; + + if (item->isnull[key_idx]) + { + all_match = false; + break; + } + + fcinfo_arr[i]->args[0].value = item->values[key_idx]; + fcinfo_arr[i]->args[0].isnull = false; + fcinfo_arr[i]->args[1].value = query_values[i]; + fcinfo_arr[i]->args[1].isnull = false; + fcinfo_arr[i]->isnull = false; + + fresult = FunctionCallInvoke(fcinfo_arr[i]); + + if (fcinfo_arr[i]->isnull || !DatumGetBool(fresult)) + { + all_match = false; + break; + } + } + + if (all_match) + total_sel += item->frequency; + } + + /* + * Non-MCV estimation for multi-key exact match. Each MCV item is a + * unique value combination, so mcvlist->nitems is the number of + * distinct combinations in the MCV. ndistinct here is the product of + * per-column ndistinct values. + */ + if (total_sel <= 0.0 && ndistinct > mcvlist->nitems) + { + double other_freq; + + other_freq = (1.0 - mcv_totalsel) / (ndistinct - mcvlist->nitems); + CLAMP_PROBABILITY(other_freq); + + total_sel = other_freq; + CLAMP_PROBABILITY(total_sel); + } + } + + return total_sel; +} + +/* + * extract_filter_info + * Extract filter column and constant value(s) from a filter clause + * + * Handles two types of filter clauses: + * 1. OpExpr: col = constant + * 2. ScalarArrayOpExpr: col IN (const1, const2, ...) + * + * Returns true if a valid filter pattern is found, false otherwise. + * On success, sets *filter_var, *filter_values (list of Datums), + * *filter_type, *collation, and *is_in_clause. + */ +static bool +extract_filter_info(Node *clause, + Index expected_relid, + Var **filter_var, + List **filter_values, + Oid *filter_type, + Oid *collation, + bool *is_in_clause) +{ + *filter_var = NULL; + *filter_values = NIL; + *is_in_clause = false; + + /* Case 1: OpExpr - simple equality (col = const) */ + if (IsA(clause, OpExpr)) + { + OpExpr *opexpr = (OpExpr *) clause; + Node *var_node = NULL; + Const *const_node = NULL; + + if (list_length(opexpr->args) != 2) + return false; + + if (!examine_opclause_args(opexpr->args, &var_node, &const_node, NULL)) + return false; + + if (!var_node || !const_node || !IsA(var_node, Var)) + return false; + + if (const_node->constisnull) + return false; + + *filter_var = (Var *) var_node; + + /* Verify the Var is from the expected relation */ + if ((*filter_var)->varno != expected_relid) + return false; + + /* Create single-element list - store Datum as pointer */ + *filter_values = list_make1(DatumGetPointer(const_node->constvalue)); + *filter_type = const_node->consttype; + *collation = opexpr->inputcollid; + *is_in_clause = false; + + return true; + } + + /* Case 2: ScalarArrayOpExpr - IN clause (col IN (...)) */ + else if (IsA(clause, ScalarArrayOpExpr)) + { + ScalarArrayOpExpr *saop = (ScalarArrayOpExpr *) clause; + Node *scalar_node; + Node *array_node; + Const *array_const; + ArrayType *arr; + int nitems; + Datum *items; + bool *nulls; + int i; + Oid elmtype; + int16 elmlen; + bool elmbyval; + char elmalign; + + /* Only support ANY (IN), not ALL */ + if (!saop->useOr) + return false; + + if (list_length(saop->args) != 2) + return false; + + scalar_node = (Node *) linitial(saop->args); + array_node = (Node *) lsecond(saop->args); + + /* Strip RelabelType */ + while (IsA(scalar_node, RelabelType)) + scalar_node = (Node *) ((RelabelType *) scalar_node)->arg; + + /* Scalar must be a Var after stripping coercions */ + if (!IsA(scalar_node, Var)) + return false; + + *filter_var = (Var *) scalar_node; + + /* Verify the Var is from the expected relation */ + if ((*filter_var)->varno != expected_relid) + return false; + + /* Array must be a Const for us to extract values */ + if (!IsA(array_node, Const)) + return false; + + array_const = (Const *) array_node; + + /* Can't handle NULL arrays */ + if (array_const->constisnull) + return false; + + /* Deconstruct the array */ + arr = DatumGetArrayTypeP(array_const->constvalue); + elmtype = ARR_ELEMTYPE(arr); + + /* Get type info for deconstruction */ + get_typlenbyvalalign(elmtype, &elmlen, &elmbyval, &elmalign); + + deconstruct_array(arr, elmtype, elmlen, elmbyval, elmalign, + &items, &nulls, &nitems); + + /* Build list of non-NULL Datums */ + *filter_values = NIL; + for (i = 0; i < nitems; i++) + { + if (!nulls[i]) + { + /* + * Store Datum as pointer - safe since Datum and pointer are + * same size + */ + *filter_values = lappend(*filter_values, DatumGetPointer(items[i])); + } + } + + /* If all values were NULL, we can't use this */ + if (*filter_values == NIL) + return false; + + *filter_type = elmtype; + *collation = saop->inputcollid; + *is_in_clause = true; + + return true; + } + + return false; +} + +/* + * match_filters_for_rel - + * Match a rel's base restrictions against a stat's stxkeys. + * + * For each restriction clause on rel that extract_filter_info can parse, + * check whether the filtered column appears in the stat's stxkeys with a + * matching keyref value. Matched filters are appended to the output lists. + * + * keyref_match is the keyrefs value that identifies this rel's columns in + * the stat: 1 for the anchor rel, != 1 for the other rel. + */ +static void +match_filters_for_rel(RelOptInfo *rel, + StatisticExtInfo *stat, + bool is_anchor, + List **stxkey_indexes, + List **filter_values, + List **covered_rinfos, + List **filter_collations) +{ + ListCell *lc; + + foreach(lc, rel->baserestrictinfo) + { + RestrictInfo *ri = lfirst_node(RestrictInfo, lc); + Var *filter_var; + List *values; + Oid filter_type; + Oid filter_collation; + bool filter_is_in; + int key_idx = -1; + int pos = 0; + ListCell *lc_attr; + ListCell *lc_ref; + + if (!extract_filter_info((Node *) ri->clause, + rel->relid, + &filter_var, &values, + &filter_type, &filter_collation, + &filter_is_in)) + continue; + + forboth(lc_attr, stat->keyattrs, lc_ref, stat->keyrefs) + { + bool ref_matches = is_anchor + ? (lfirst_int(lc_ref) == 1) + : (lfirst_int(lc_ref) != 1); + + if (lfirst_int(lc_attr) == filter_var->varattno && ref_matches) + { + key_idx = pos; + break; + } + pos++; + } + + if (key_idx < 0) + continue; + + *stxkey_indexes = lappend_int(*stxkey_indexes, key_idx); + *filter_values = lappend(*filter_values, values); + *covered_rinfos = lappend(*covered_rinfos, ri); + *filter_collations = lappend_oid(*filter_collations, filter_collation); + } +} + +/* + * match_join_stat - + * Check if a stat matches the join clause and find covered filters. + * + * Checks whether the stat's join condition matches join_rinfo, then walks + * other_rel's baserestrictinfo to find filter clauses covered by the stat's + * stxkeys. Uncovered filters are skipped; they are already reflected in + * other_rel->rows. + * + * Returns the number of covered filters (0 means no match -- either the join + * clause didn't match or no filters were covered). On success, populates + * output parameters with the covered filter details needed by + * compute_join_mcv_selec(). + */ +static int +match_join_stat(StatisticExtInfo *stat, + RelOptInfo *anchor_rel, + RelOptInfo *other_rel, + RestrictInfo *join_rinfo, + List **covered_rinfos, + List **stxkey_indexes, + List **filter_values, + List **filter_collations) +{ + OpExpr *stat_op; + Var *stat_left_var; + Var *stat_right_var; + AttrNumber anchor_joinkey; + AttrNumber other_joinkey; + Oid stat_join_opno; /* operator normalized to (anchor, other) */ + bool join_matched = false; + + *covered_rinfos = NIL; + *stxkey_indexes = NIL; + *filter_values = NIL; + *filter_collations = NIL; + + /* + * Check if the join clause matches this stat's join condition. + * + * XXX Currently only single-condition joins are supported. To handle + * multiple join conditions on the same baserel pair (e.g., a.x = b.x AND + * a.y = b.y), the caller would need to group clauses by baserel pair and + * present the full group here, and this function would need to match all + * conditions against the stat's joinconds. The sampling code in ANALYZE + * would also need to support composite join key lookups. + */ + if (list_length(stat->joinconds) != 1) + return 0; + + stat_op = (OpExpr *) linitial(stat->joinconds); + Assert(IsA(stat_op, OpExpr) && list_length(stat_op->args) == 2); + stat_left_var = (Var *) linitial(stat_op->args); + stat_right_var = (Var *) lsecond(stat_op->args); + Assert(IsA(stat_left_var, Var) && IsA(stat_right_var, Var)); + + /* + * Identify anchor and other join key attnums from the stat's joincond, + * and normalize the operator to (anchor, other) argument order. + */ + if (stat_left_var->varno == 1 && stat_right_var->varno == 2) + { + anchor_joinkey = stat_left_var->varattno; + other_joinkey = stat_right_var->varattno; + stat_join_opno = stat_op->opno; + } + else if (stat_left_var->varno == 2 && stat_right_var->varno == 1) + { + anchor_joinkey = stat_right_var->varattno; + other_joinkey = stat_left_var->varattno; + stat_join_opno = get_commutator(stat_op->opno); + if (!OidIsValid(stat_join_opno)) + return 0; + } + else + return 0; + + if (IsA(join_rinfo->clause, OpExpr)) + { + OpExpr *opexpr = (OpExpr *) join_rinfo->clause; + + if (list_length(opexpr->args) == 2) + { + Node *left_node = linitial(opexpr->args); + Node *right_node = lsecond(opexpr->args); + + while (IsA(left_node, RelabelType)) + left_node = (Node *) ((RelabelType *) left_node)->arg; + while (IsA(right_node, RelabelType)) + right_node = (Node *) ((RelabelType *) right_node)->arg; + + if (IsA(left_node, Var) && IsA(right_node, Var)) + { + Var *left_var = (Var *) left_node; + Var *right_var = (Var *) right_node; + + if (bms_is_member(left_var->varno, anchor_rel->relids) && + bms_is_member(right_var->varno, other_rel->relids) && + left_var->varattno == anchor_joinkey && + right_var->varattno == other_joinkey && + opexpr->opno == stat_join_opno) + join_matched = true; + + if (bms_is_member(right_var->varno, anchor_rel->relids) && + bms_is_member(left_var->varno, other_rel->relids) && + right_var->varattno == anchor_joinkey && + left_var->varattno == other_joinkey && + opexpr->opno == get_commutator(stat_join_opno)) + join_matched = true; + } + } + } + + if (!join_matched) + return 0; + + /* + * Find filter clauses on other_rel and anchor_rel that are covered by + * this stat's stxkeys. Without at least one covered filter, the stat + * cannot improve on the standard join selectivity estimate, so bail out. + * + * For other_rel filters, match stxkeys columns with keyrefs != 1 (joined + * relation columns). For anchor_rel filters, match stxkeys columns with + * keyrefs == 1 (anchor relation columns). + */ + match_filters_for_rel(other_rel, stat, false, + stxkey_indexes, filter_values, + covered_rinfos, filter_collations); + match_filters_for_rel(anchor_rel, stat, true, + stxkey_indexes, filter_values, + covered_rinfos, filter_collations); + + return list_length(*covered_rinfos); +} + +/* + * compute_join_mcv_selec - + * Compute join MCV selectivity from a matched stat. + * + * Given a stat and the filter matching results from match_join_stat(), + * loads the MCVList and computes the raw join+filter selectivity. + * stxkey_indexes contains 0-based stxkey positions produced by + * match_join_stat(). + * + * ndistinct is the number of distinct values for the filter column, + * used for non-MCV estimation. Pass 0 to disable estimation. + * + * Returns selectivity >= 0. + */ +static Selectivity +compute_join_mcv_selec(StatisticExtInfo *stat, + List *stxkey_indexes, + List *filter_values, + List *filter_collations, + double ndistinct) +{ + MCVList *mcvlist; + List *flat_filter_values; + Selectivity selec; + + /* Load the MCVList */ + mcvlist = statext_mcv_load(stat->statOid, false); + if (!mcvlist) + return 0.0; + + /* Flatten filter values for the selectivity function */ + flat_filter_values = (list_length(filter_values) == 1) ? + linitial(filter_values) : filter_values; + + selec = join_mcv_clauselist_selectivity(mcvlist, + flat_filter_values, + stxkey_indexes, filter_collations, + ndistinct); + + return selec; +} + +/* + * join_mcv_clause_selectivity + * Estimate a join clause's selectivity using join MCV statistics. + * + * Given a join clause (e.g., a.id = b.id), extracts the baserel pair from + * the Vars and searches their statlists for a join stat covering that pair. + * This per baserel pair lookup produces the same result regardless of which + * component rel pair is used to form the joinrel first. + * + * The stat may cover only a subset of the filter clauses on either rel. + * The raw MCV selectivity is P(join AND covered_filters) relative to + * anchor_totalrows. Since the planner's join size formula is: + * + * nrows = anchor_rows * other_rows * sel(clause) + * + * and both anchor_rows and other_rows already reflect their respective base + * restrictions (including uncovered filters), we convert via: + * + * adjusted_sel = raw_sel + * / (covered_anchor_sel * other_tuples * covered_other_sel) + * + * where covered_anchor_sel and covered_other_sel are computed separately + * through clauselist_selectivity so that single-table extended stats on + * each rel are used, ensuring consistency with how each rel's rows was + * computed. Uncovered filters are accounted for through the respective + * rel's rows in the planner's formula. + * + * On success, returns the adjusted selectivity (> 0). + * Returns 0 if no applicable join stat is found. + */ +Selectivity +join_mcv_clause_selectivity(PlannerInfo *root, + RestrictInfo *rinfo) +{ + OpExpr *opexpr; + Node *left_node; + Node *right_node; + Var *left_var; + Var *right_var; + int varno1; + int varno2; + RelOptInfo *baserel1; + RelOptInfo *baserel2; + RelOptInfo *rels[2]; + int r; + Selectivity best_sel = 0.0; + int best_ncovered = 0; + bool found_full_coverage = false; + SpecialJoinInfo sjinfo_data; + + /* Must be a simple Var = Var equality */ + if (!IsA(rinfo->clause, OpExpr)) + return 0.0; + + opexpr = (OpExpr *) rinfo->clause; + if (list_length(opexpr->args) != 2) + return 0.0; + + /* Strip RelabelType wrappers */ + left_node = linitial(opexpr->args); + right_node = lsecond(opexpr->args); + while (IsA(left_node, RelabelType)) + left_node = (Node *) ((RelabelType *) left_node)->arg; + while (IsA(right_node, RelabelType)) + right_node = (Node *) ((RelabelType *) right_node)->arg; + if (!IsA(left_node, Var) || !IsA(right_node, Var)) + return 0.0; + + left_var = (Var *) left_node; + right_var = (Var *) right_node; + + varno1 = left_var->varno; + varno2 = right_var->varno; + if (varno1 == varno2) + return 0.0; + + baserel1 = find_base_rel(root, varno1); + baserel2 = find_base_rel(root, varno2); + rels[0] = baserel1; + rels[1] = baserel2; + + /* + * Build a dummy SpecialJoinInfo so that clause_selectivity_ext treats the + * join clause as a join (dispatching to eqjoinsel) rather than as a + * restriction (dispatching to eqsel). + */ + init_dummy_sjinfo(&sjinfo_data, baserel1->relids, baserel2->relids); + + /* + * Try each base rel as the potential stat anchor. A join stat is stored + * on its anchor table's statlist (stxrelid = anchor). We prefer stats + * covering more filter columns; a full-coverage stat is returned + * immediately, while partial-coverage results are saved and used only if + * no better stat is found. + */ + for (r = 0; r < 2; r++) + { + RelOptInfo *anchor_rel = rels[r]; + RelOptInfo *other_rel = rels[1 - r]; + Oid other_reloid; + RangeTblEntry *other_rte; + ListCell *lc; + int nfilters; /* total baserestrictinfo clauses on both rels */ + + other_rte = root->simple_rte_array[other_rel->relid]; + other_reloid = other_rte->relid; + nfilters = list_length(other_rel->baserestrictinfo) + + list_length(anchor_rel->baserestrictinfo); + + foreach(lc, anchor_rel->statlist) + { + StatisticExtInfo *stat = (StatisticExtInfo *) lfirst(lc); + List *covered_rinfos = NIL; + List *anchor_covered_rinfos = NIL; + List *other_covered_rinfos = NIL; + List *stxkey_indexes = NIL; + List *filter_values = NIL; + List *filter_collations = NIL; + Selectivity raw_sel; + Selectivity covered_anchor_sel = 1.0; + Selectivity covered_other_sel = 1.0; + int ncovered; /* number of filters the stat covers */ + ListCell *lc2; + double ndistinct = 0; + Selectivity standard_sel; + + /* Skip non-join and non-MCV stats */ + if (stat->joinrels == NIL || stat->kind != STATS_EXT_MCV) + continue; + + /* Check that this stat joins to the other relation */ + if (!list_member_oid(stat->joinrels, other_reloid)) + continue; + + /* Does this stat match our join clause and cover any filters? */ + ncovered = match_join_stat(stat, anchor_rel, other_rel, rinfo, + &covered_rinfos, &stxkey_indexes, + &filter_values, &filter_collations); + if (ncovered == 0) + continue; + + /* Split covered filters by relation */ + foreach(lc2, covered_rinfos) + { + RestrictInfo *ri = lfirst_node(RestrictInfo, lc2); + + if (bms_is_member(anchor_rel->relid, ri->clause_relids)) + anchor_covered_rinfos = lappend(anchor_covered_rinfos, ri); + else + other_covered_rinfos = lappend(other_covered_rinfos, ri); + } + + /* + * Look up ndistinct for each covered filter column, used for + * non-MCV estimation. For single-key, this is the per-column + * ndistinct. For multi-key, it is the product of per-column + * ndistinct values (independence assumption). Pass 0 to disable + * correction when stats are unavailable. + */ + if (covered_rinfos != NIL) + { + ndistinct = 1.0; + + foreach(lc2, covered_rinfos) + { + RestrictInfo *filter_ri = lfirst_node(RestrictInfo, + lc2); + Var *filter_var = NULL; + List *dummy_values; + Oid dummy_type; + Oid dummy_collation; + bool dummy_is_in; + RelOptInfo *filter_rel; + VariableStatData vardata; + bool isdefault; + double col_ndistinct; + + if (bms_is_member(other_rel->relid, + filter_ri->clause_relids)) + filter_rel = other_rel; + else + filter_rel = anchor_rel; + + if (!extract_filter_info((Node *) filter_ri->clause, + filter_rel->relid, + &filter_var, &dummy_values, + &dummy_type, + &dummy_collation, + &dummy_is_in) || + filter_var == NULL) + { + ndistinct = 0; + break; + } + + examine_variable(root, (Node *) filter_var, 0, + &vardata); + col_ndistinct = get_variable_numdistinct(&vardata, + &isdefault); + ReleaseVariableStats(vardata); + + if (isdefault) + { + ndistinct = 0; + break; + } + + ndistinct *= col_ndistinct; + } + } + + /* Compute raw MCV selectivity (join AND covered filters) */ + raw_sel = compute_join_mcv_selec(stat, stxkey_indexes, + filter_values, + filter_collations, + ndistinct); + + /* + * Full coverage with zero MCV match means the stat saw the filter + * combination and found no matches. This is authoritative -- do + * not let a partial-coverage stat override it with a positive + * estimate. + */ + if (ncovered >= nfilters && raw_sel <= 0) + { + found_full_coverage = true; + continue; + } + + if (raw_sel <= 0) + continue; + + /* + * Convert from anchor-relative frequency to join selectivity. + * + * raw_sel is P(join AND covered_filters) / anchor_totalrows, + * computed from the MCV frequencies. The planner computes: + * + * nrows = anchor_rows * other_rows * join_sel + * + * where anchor_rows and other_rows each already reflect their + * base restrictions. If we return raw_sel directly, covered + * filters would be double-counted: anchor-side covered filters + * appear in both raw_sel and anchor_rows, and other-side covered + * filters appear in both raw_sel and other_rows. To cancel both: + * + * adjusted_sel = raw_sel / (covered_anchor_sel * other_tuples * + * covered_other_sel) + * + * The other_tuples * covered_other_sel denominator is clamped to + * at least 1.0 to match clamp_row_est() in + * set_baserel_size_estimates. Selectivities are computed via + * clauselist_selectivity so that single-table extended stats are + * used, ensuring consistency with how each rel's rows was + * computed. + */ + if (anchor_covered_rinfos != NIL) + covered_anchor_sel = clauselist_selectivity(root, + anchor_covered_rinfos, + 0, + JOIN_INNER, + NULL); + if (other_covered_rinfos != NIL) + covered_other_sel = clauselist_selectivity(root, + other_covered_rinfos, + 0, + JOIN_INNER, + NULL); + if (covered_anchor_sel > 0 && other_rel->tuples > 0) + { + double other_denom = Max(other_rel->tuples * covered_other_sel, + 1.0); + + raw_sel /= covered_anchor_sel * other_denom; + CLAMP_PROBABILITY(raw_sel); + } + else + continue; + + /* + * Cross-check: the MCV-based estimate shouldn't be lower than + * what the standard join estimator would produce (eqjoinsel on + * per-column stats only, no extended stats). If our estimate is + * lower, skip this stat and fall back to the standard estimate. + */ + standard_sel = clause_selectivity_ext(root, + (Node *) rinfo->clause, + 0, + JOIN_INNER, + &sjinfo_data, + false); + if (raw_sel < standard_sel) + continue; + + /* Full coverage: return immediately */ + if (ncovered >= nfilters) + return raw_sel; + + /* Partial coverage: save if better than previous best */ + if (ncovered > best_ncovered) + { + /* + * Skip partial-coverage results when other_rel->rows is at + * the clamp_row_est floor. This means the single-table stats + * predict ~0 matching rows for the full filter set. The + * partial-coverage formula would inflate the estimate because + * it divides by only the covered filter selectivity, while + * the uncovered filters (which drive rows toward zero) are + * accounted for only through the clamped rows value. + */ + if (other_rel->rows <= 1.0 || anchor_rel->rows <= 1.0) + continue; + + best_sel = raw_sel; + best_ncovered = ncovered; + } + } + } + + /* + * Do not use a partial-coverage result if a full-coverage stat already + * reported zero MCV match -- the full stat's answer is more + * authoritative. + */ + if (best_sel > 0 && !found_full_coverage) + return best_sel; + + return 0.0; +} diff --git a/src/backend/statistics/meson.build b/src/backend/statistics/meson.build index 9a7bf55e301..450b5ef0c64 100644 --- a/src/backend/statistics/meson.build +++ b/src/backend/statistics/meson.build @@ -5,6 +5,7 @@ backend_sources += files( 'dependencies.c', 'extended_stats.c', 'extended_stats_funcs.c', + 'join_mcv.c', 'mcv.c', 'mvdistinct.c', 'relation_stats.c', diff --git a/src/backend/tcop/utility.c b/src/backend/tcop/utility.c index 73a56f1df1d..9faf47cb66a 100644 --- a/src/backend/tcop/utility.c +++ b/src/backend/tcop/utility.c @@ -1890,28 +1890,44 @@ ProcessUtilitySlow(ParseState *pstate, { Oid relid; CreateStatsStmt *stmt = (CreateStatsStmt *) parsetree; - RangeVar *rel = (RangeVar *) linitial(stmt->relations); - - if (!IsA(rel, RangeVar)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("CREATE STATISTICS only supports relation names in the FROM clause"))); + Node *fromNode = (Node *) linitial(stmt->relations); /* - * CREATE STATISTICS will influence future execution plans - * but does not interfere with currently executing plans. - * So it should be enough to take ShareUpdateExclusiveLock - * on relation, conflicting with ANALYZE and other DDL - * that sets statistical information, but not with normal - * queries. - * - * XXX RangeVarCallbackOwnsRelation not needed here, to - * keep the same behavior as before. + * For single-table stats: FROM clause is a simple + * RangeVar, for join stats: FROM clause is a JoinExpr */ - relid = RangeVarGetRelid(rel, ShareUpdateExclusiveLock, false); + if (IsA(fromNode, RangeVar)) + { + RangeVar *rel = (RangeVar *) fromNode; - /* Run parse analysis ... */ - stmt = transformStatsStmt(relid, stmt, queryString); + /* + * CREATE STATISTICS will influence future execution + * plans but does not interfere with currently + * executing plans. So it should be enough to take + * ShareUpdateExclusiveLock on relation, conflicting + * with ANALYZE and other DDL that sets statistical + * information, but not with normal queries. + * + * XXX RangeVarCallbackOwnsRelation not needed here, + * to keep the same behavior as before. + */ + relid = RangeVarGetRelid(rel, ShareUpdateExclusiveLock, false); + + /* Run parse analysis ... */ + stmt = transformStatsStmt(relid, stmt, queryString); + } + else if (IsA(fromNode, JoinExpr)) + { + /* Join statistics passes 0 as relid */ + relid = InvalidOid; + + /* Run parse analysis ... */ + stmt = transformStatsStmt(relid, stmt, queryString); + } + else + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("CREATE STATISTICS only supports relation names or JOIN clauses in the FROM clause"))); address = CreateStatistics(stmt, true); } diff --git a/src/backend/utils/adt/ruleutils.c b/src/backend/utils/adt/ruleutils.c index c781cdc84d3..360a230c99f 100644 --- a/src/backend/utils/adt/ruleutils.c +++ b/src/backend/utils/adt/ruleutils.c @@ -369,7 +369,13 @@ static char *pg_get_indexdef_worker(Oid indexrelid, int colno, static void make_propgraphdef_elements(StringInfo buf, Oid pgrelid, char pgekind); static void make_propgraphdef_labels(StringInfo buf, Oid elid, const char *elalias, Oid elrelid); static void make_propgraphdef_properties(StringInfo buf, Oid ellabelid, Oid elrelid); -static char *pg_get_statisticsobj_worker(Oid statextid, bool columns_only, + +/* Worker modes for pg_get_statisticsobj_worker */ +#define STATS_DEF_FULL 0 /* full CREATE STATISTICS DDL */ +#define STATS_DEF_COLUMNS_ONLY 1 /* columns/expressions only */ +#define STATS_DEF_COLUMNS_FROM 2 /* columns + FROM clause */ + +static char *pg_get_statisticsobj_worker(Oid statextid, int defmode, bool missing_ok); static char *pg_get_partkeydef_worker(Oid relid, int prettyFlags, bool attrsOnly, bool missing_ok); @@ -1969,7 +1975,7 @@ pg_get_statisticsobjdef(PG_FUNCTION_ARGS) Oid statextid = PG_GETARG_OID(0); char *res; - res = pg_get_statisticsobj_worker(statextid, false, true); + res = pg_get_statisticsobj_worker(statextid, STATS_DEF_FULL, true); if (res == NULL) PG_RETURN_NULL(); @@ -1984,7 +1990,7 @@ pg_get_statisticsobjdef(PG_FUNCTION_ARGS) char * pg_get_statisticsobjdef_string(Oid statextid) { - return pg_get_statisticsobj_worker(statextid, false, false); + return pg_get_statisticsobj_worker(statextid, STATS_DEF_FULL, false); } /* @@ -1997,7 +2003,29 @@ pg_get_statisticsobjdef_columns(PG_FUNCTION_ARGS) Oid statextid = PG_GETARG_OID(0); char *res; - res = pg_get_statisticsobj_worker(statextid, true, true); + res = pg_get_statisticsobj_worker(statextid, STATS_DEF_COLUMNS_ONLY, true); + + if (res == NULL) + PG_RETURN_NULL(); + + PG_RETURN_TEXT_P(string_to_text(res)); +} + +/* + * pg_get_statisticsobjdef_columns_from + * Get the columns and FROM clause of an extended statistics definition, + * without the CREATE STATISTICS name (kinds) prefix. + * + * For regular stats: "a, b FROM tablename" + * For join stats: "t1.col FROM t2 t2 JOIN t1 t1 ON (t2.fk = t1.id)" + */ +Datum +pg_get_statisticsobjdef_columns_from(PG_FUNCTION_ARGS) +{ + Oid statextid = PG_GETARG_OID(0); + char *res; + + res = pg_get_statisticsobj_worker(statextid, STATS_DEF_COLUMNS_FROM, true); if (res == NULL) PG_RETURN_NULL(); @@ -2009,7 +2037,7 @@ pg_get_statisticsobjdef_columns(PG_FUNCTION_ARGS) * Internal workhorse to decompile an extended statistics object. */ static char * -pg_get_statisticsobj_worker(Oid statextid, bool columns_only, bool missing_ok) +pg_get_statisticsobj_worker(Oid statextid, int defmode, bool missing_ok) { Form_pg_statistic_ext statextrec; HeapTuple statexttup; @@ -2027,8 +2055,17 @@ pg_get_statisticsobj_worker(Oid statextid, bool columns_only, bool missing_ok) ListCell *lc; List *exprs = NIL; bool has_exprs; + bool is_join_stat; int ncolumns; + /* Join stat fields */ + oidvector *joinrels = NULL; + int2vector *keyrefs = NULL; + List *joinconds = NIL; + Oid *all_reloids = NULL; + char **all_aliases = NULL; + int nrels = 0; + statexttup = SearchSysCache1(STATEXTOID, ObjectIdGetDatum(statextid)); if (!HeapTupleIsValid(statexttup)) @@ -2041,8 +2078,78 @@ pg_get_statisticsobj_worker(Oid statextid, bool columns_only, bool missing_ok) /* has the statistics expressions? */ has_exprs = !heap_attisnull(statexttup, Anum_pg_statistic_ext_stxexprs, NULL); + /* is this a join statistics object? */ + is_join_stat = !heap_attisnull(statexttup, Anum_pg_statistic_ext_stxjoinrels, NULL); + statextrec = (Form_pg_statistic_ext) GETSTRUCT(statexttup); + /* + * For join statistics, load the join-related catalog fields so we can + * emit table-qualified column names and the JOIN ... ON FROM clause. + */ + if (is_join_stat) + { + Datum jdatum; + bool isnull; + char *str; + + jdatum = SysCacheGetAttr(STATEXTOID, statexttup, + Anum_pg_statistic_ext_stxjoinrels, &isnull); + Assert(!isnull); + joinrels = (oidvector *) DatumGetPointer(jdatum); + + jdatum = SysCacheGetAttr(STATEXTOID, statexttup, + Anum_pg_statistic_ext_stxkeyrefs, &isnull); + Assert(!isnull); + keyrefs = (int2vector *) DatumGetPointer(jdatum); + + jdatum = SysCacheGetAttr(STATEXTOID, statexttup, + Anum_pg_statistic_ext_stxjoinconds, &isnull); + Assert(!isnull); + str = TextDatumGetCString(jdatum); + joinconds = (List *) stringToNode(str); + pfree(str); + + /* Build relation OID and alias arrays: [stxrelid, joinrels...] */ + nrels = 1 + joinrels->dim1; + all_reloids = palloc(nrels * sizeof(Oid)); + all_aliases = palloc(nrels * sizeof(char *)); + all_reloids[0] = statextrec->stxrelid; + all_aliases[0] = get_relation_name(statextrec->stxrelid); + for (i = 0; i < joinrels->dim1; i++) + { + all_reloids[i + 1] = joinrels->values[i]; + all_aliases[i + 1] = get_relation_name(joinrels->values[i]); + } + + /* + * Disambiguate aliases when different relations share the same + * unqualified name (e.g., s1.t and s2.t both get alias "t"). + * Append _1, _2, ... following the set_rtable_names() convention. + */ + for (i = 1; i < nrels; i++) + { + int j; + int suffix = 0; + + for (j = 0; j < i; j++) + { + if (strcmp(all_aliases[i], all_aliases[j]) == 0) + { + char buf[NAMEDATALEN]; + + suffix++; + snprintf(buf, sizeof(buf), "%s_%d", + get_relation_name(all_reloids[i]), suffix); + all_aliases[i] = pstrdup(buf); + + /* Restart scan to check for further conflicts */ + j = -1; + } + } + } + } + /* * Get the statistics expressions, if any. (NOTE: we do not use the * relcache versions of the expressions, because we want to display @@ -2067,7 +2174,7 @@ pg_get_statisticsobj_worker(Oid statextid, bool columns_only, bool missing_ok) initStringInfo(&buf); - if (!columns_only) + if (defmode == STATS_DEF_FULL) { nsp = get_namespace_name_or_temp(statextrec->stxnamespace); appendStringInfo(&buf, "CREATE STATISTICS %s", @@ -2115,7 +2222,7 @@ pg_get_statisticsobj_worker(Oid statextid, bool columns_only, bool missing_ok) * kinds. */ if ((!ndistinct_enabled || !dependencies_enabled || !mcv_enabled) && - (ncolumns > 1)) + (ncolumns > 1 || is_join_stat)) { bool gotone = false; @@ -2151,9 +2258,21 @@ pg_get_statisticsobj_worker(Oid statextid, bool columns_only, bool missing_ok) if (colno > 0) appendStringInfoString(&buf, ", "); - attname = get_attname(statextrec->stxrelid, attnum, false); + if (is_join_stat) + { + /* Join stat: table-qualify column with its relation's alias */ + int relidx = keyrefs->values[colno] - 1; - appendStringInfoString(&buf, quote_identifier(attname)); + attname = get_attname(all_reloids[relidx], attnum, false); + appendStringInfo(&buf, "%s.%s", + quote_identifier(all_aliases[relidx]), + quote_identifier(attname)); + } + else + { + attname = get_attname(statextrec->stxrelid, attnum, false); + appendStringInfoString(&buf, quote_identifier(attname)); + } } context = deparse_context_for(get_relation_name(statextrec->stxrelid), @@ -2180,9 +2299,66 @@ pg_get_statisticsobj_worker(Oid statextid, bool columns_only, bool missing_ok) colno++; } - if (!columns_only) - appendStringInfo(&buf, " FROM %s", - generate_relation_name(statextrec->stxrelid, NIL)); + if (defmode != STATS_DEF_COLUMNS_ONLY) + { + if (is_join_stat) + { + /* + * Emit FROM ... JOIN ... ON syntax for join statistics. + * + * Use schema-qualified relation names with unqualified table + * names as aliases, matching how the column references above are + * emitted. + */ + appendStringInfo(&buf, " FROM %s %s", + generate_relation_name(statextrec->stxrelid, NIL), + quote_identifier(all_aliases[0])); + + for (i = 0; i < joinrels->dim1; i++) + { + appendStringInfo(&buf, " JOIN %s %s ON (", + generate_relation_name(joinrels->values[i], NIL), + quote_identifier(all_aliases[i + 1])); + + /* Find the join condition for this relation (varno = i+2) */ + foreach(lc, joinconds) + { + OpExpr *op = (OpExpr *) lfirst(lc); + Var *lvar = (Var *) linitial(op->args); + Var *rvar = (Var *) lsecond(op->args); + + if (lvar->varno == i + 2 || rvar->varno == i + 2) + { + int lrelidx = lvar->varno - 1; + int rrelidx = rvar->varno - 1; + char *lcolname; + char *rcolname; + + lcolname = get_attname(all_reloids[lrelidx], + lvar->varattno, false); + rcolname = get_attname(all_reloids[rrelidx], + rvar->varattno, false); + appendStringInfo(&buf, "%s.%s %s %s.%s", + quote_identifier(all_aliases[lrelidx]), + quote_identifier(lcolname), + generate_operator_name(op->opno, + lvar->vartype, + rvar->vartype), + quote_identifier(all_aliases[rrelidx]), + quote_identifier(rcolname)); + break; + } + } + + appendStringInfoChar(&buf, ')'); + } + } + else + { + appendStringInfo(&buf, " FROM %s", + generate_relation_name(statextrec->stxrelid, NIL)); + } + } ReleaseSysCache(statexttup); diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c index e1449654f96..ff8fcbbe1f2 100644 --- a/src/bin/psql/describe.c +++ b/src/bin/psql/describe.c @@ -2920,8 +2920,14 @@ describeOneTableDetails(const char *schemaname, " " CppAsString2(STATS_EXT_NDISTINCT) " = any(stxkind) AS ndist_enabled,\n" " " CppAsString2(STATS_EXT_DEPENDENCIES) " = any(stxkind) AS deps_enabled,\n" " " CppAsString2(STATS_EXT_MCV) " = any(stxkind) AS mcv_enabled,\n" - "stxstattarget\n" - "FROM pg_catalog.pg_statistic_ext\n" + "stxstattarget,\n"); + if (pset.sversion >= 190000) + appendPQExpBufferStr(&buf, + "pg_catalog.pg_get_statisticsobjdef_columns_from(oid) AS body\n"); + else + appendPQExpBufferStr(&buf, + "NULL AS body\n"); + appendPQExpBuffer(&buf, "FROM pg_catalog.pg_statistic_ext\n" "WHERE stxrelid = '%s'\n" "ORDER BY nsp, stxname;", oid); @@ -2992,9 +2998,18 @@ describeOneTableDetails(const char *schemaname, appendPQExpBufferChar(&buf, ')'); } - appendPQExpBuffer(&buf, " ON %s FROM %s", - PQgetvalue(result, i, 4), - PQgetvalue(result, i, 1)); + /* + * Use pg_get_statisticsobjdef_columns_from() when available + * (PG19+); fall back to simple column list for older + * servers. + */ + if (!PQgetisnull(result, i, 9)) + appendPQExpBuffer(&buf, " ON %s", + PQgetvalue(result, i, 9)); + else + appendPQExpBuffer(&buf, " ON %s FROM %s", + PQgetvalue(result, i, 4), + PQgetvalue(result, i, 1)); /* Show the stats target if it's not default */ if (!PQgetisnull(result, i, 8) && @@ -3025,9 +3040,15 @@ describeOneTableDetails(const char *schemaname, " " CppAsString2(STATS_EXT_MCV) " = any(stxkind) AS mcv_enabled,\n"); if (pset.sversion >= 130000) - appendPQExpBufferStr(&buf, " stxstattarget\n"); + appendPQExpBufferStr(&buf, " stxstattarget,\n"); + else + appendPQExpBufferStr(&buf, " -1 AS stxstattarget,\n"); + if (pset.sversion >= 190000) + appendPQExpBufferStr(&buf, + " pg_catalog.pg_get_statisticsobjdef_columns_from(oid) AS body\n"); else - appendPQExpBufferStr(&buf, " -1 AS stxstattarget\n"); + appendPQExpBufferStr(&buf, + " NULL AS body\n"); appendPQExpBuffer(&buf, "FROM pg_catalog.pg_statistic_ext\n" "WHERE stxrelid = '%s'\n" "ORDER BY 1;", @@ -3072,9 +3093,21 @@ describeOneTableDetails(const char *schemaname, appendPQExpBuffer(&buf, "%smcv", gotone ? ", " : ""); } - appendPQExpBuffer(&buf, ") ON %s FROM %s", - PQgetvalue(result, i, 4), - PQgetvalue(result, i, 1)); + /* + * Use pg_get_statisticsobjdef_columns_from() when available + * (PG19+); fall back to simple column list for older + * servers. + */ + if (!PQgetisnull(result, i, 9)) + { + appendPQExpBufferChar(&buf, ')'); + appendPQExpBuffer(&buf, " ON %s", + PQgetvalue(result, i, 9)); + } + else + appendPQExpBuffer(&buf, ") ON %s FROM %s", + PQgetvalue(result, i, 4), + PQgetvalue(result, i, 1)); /* Show the stats target if it's not default */ if (strcmp(PQgetvalue(result, i, 8), "-1") != 0) @@ -5133,7 +5166,11 @@ listExtendedStats(const char *pattern, bool verbose) gettext_noop("Schema"), gettext_noop("Name")); - if (pset.sversion >= 140000) + if (pset.sversion >= 190000) + appendPQExpBuffer(&buf, + "pg_catalog.pg_get_statisticsobjdef_columns_from(es.oid) AS \"%s\"", + gettext_noop("Definition")); + else if (pset.sversion >= 140000) appendPQExpBuffer(&buf, "pg_catalog.format('%%s FROM %%s', \n" " pg_catalog.pg_get_statisticsobjdef_columns(es.oid), \n" diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index 1602962dbe1..ef0a3b1ff66 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -57,6 +57,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 202604061 +#define CATALOG_VERSION_NO 202604251 #endif diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index fa9ae79082b..f25b7c5adc1 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -4005,6 +4005,10 @@ proname => 'pg_get_statisticsobjdef_columns', provolatile => 's', prorettype => 'text', proargtypes => 'oid', prosrc => 'pg_get_statisticsobjdef_columns' }, +{ oid => '9877', descr => 'extended statistics columns and sources', + proname => 'pg_get_statisticsobjdef_columns_from', provolatile => 's', + prorettype => 'text', proargtypes => 'oid', + prosrc => 'pg_get_statisticsobjdef_columns_from' }, { oid => '6173', descr => 'extended statistics expressions', proname => 'pg_get_statisticsobjdef_expressions', provolatile => 's', prorettype => '_text', proargtypes => 'oid', diff --git a/src/include/catalog/pg_statistic_ext.h b/src/include/catalog/pg_statistic_ext.h index e4a0cb4d41c..2d21365bd98 100644 --- a/src/include/catalog/pg_statistic_ext.h +++ b/src/include/catalog/pg_statistic_ext.h @@ -59,6 +59,15 @@ CATALOG(pg_statistic_ext,3381,StatisticExtRelationId) pg_node_tree stxexprs; /* A list of expression trees for stats * attributes that are not simple column * references. */ + + /* Fields for join statistics (all NULL for single-table stats) */ + oidvector stxjoinrels; /* other participating relation OIDs */ + int2vector stxkeyrefs; /* parallel to stxkeys: 1-based varno of each + * key's source relation (1=stxrelid, + * 2=stxjoinrels[0], ...) */ + pg_node_tree stxjoinconds; /* join conditions as a List of OpExpr nodes; + * Var varnos are 1-based (1=stxrelid, + * 2=stxjoinrels[0], ...) */ #endif } FormData_pg_statistic_ext; @@ -81,6 +90,12 @@ DECLARE_INDEX(pg_statistic_ext_relid_index, 3379, StatisticExtRelidIndexId, pg_s MAKE_SYSCACHE(STATEXTOID, pg_statistic_ext_oid_index, 4); MAKE_SYSCACHE(STATEXTNAMENSP, pg_statistic_ext_name_index, 4); +/* + * For single-table statistics, all stxkeys reference stxrelid's columns. + * For join statistics (stxjoinrels IS NOT NULL), some stxkeys reference + * columns from the joined tables instead; the oidjoins test accounts for + * this by skipping join stats rows. + */ DECLARE_ARRAY_FOREIGN_KEY((stxrelid, stxkeys), pg_attribute, (attrelid, attnum)); #ifdef EXPOSE_TO_CLIENT_CODE diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index 91377a6cde3..efd240bf39c 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -3635,6 +3635,11 @@ typedef struct CreateStatsStmt char *stxcomment; /* comment to apply to stats, or NULL */ bool transformed; /* true when transformStatsStmt is finished */ bool if_not_exists; /* do nothing if stats name already exists */ + + /* Join statistics fields, populated by transformStatsStmt */ + Oid stxrelid; /* anchor table OID (InvalidOid if not join) */ + List *stxjoinrels; /* other table OIDs (list of Oid) */ + List *stxjoinconds; /* equijoin conditions (list of OpExpr) */ } CreateStatsStmt; /* diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index 693b879f76d..da2e25ce751 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -1530,6 +1530,12 @@ typedef struct StatisticExtInfo /* expressions */ List *exprs; + + /* Join statistics fields (all NIL for single-table stats) */ + List *joinrels; /* OIDs of other participating relations */ + List *keyattrs; /* raw stxkeys as list of AttrNumbers */ + List *keyrefs; /* per-key table ref (1=anchor, 2+=joined) */ + List *joinconds; /* parsed join conditions (List of OpExpr) */ } StatisticExtInfo; /* diff --git a/src/include/statistics/extended_stats_internal.h b/src/include/statistics/extended_stats_internal.h index c775442f2ee..7de67f3aebf 100644 --- a/src/include/statistics/extended_stats_internal.h +++ b/src/include/statistics/extended_stats_internal.h @@ -68,7 +68,6 @@ typedef struct StatsBuildData bool **nulls; } StatsBuildData; - extern MVNDistinct *statext_ndistinct_build(double totalrows, StatsBuildData *data); extern bytea *statext_ndistinct_serialize(MVNDistinct *ndistinct); extern MVNDistinct *statext_ndistinct_deserialize(bytea *data); @@ -96,6 +95,24 @@ extern Datum statext_mcv_import(int elevel, int numattrs, Oid *atttypids, bool *mcv_nulls, float8 *freqs, float8 *base_freqs); +extern Oid find_index_for_operator(Relation rel, AttrNumber attnum, + Oid eq_op); + +extern MCVList *statext_join_mcv_build(Relation anchor_rel, + Oid *joinrel_oids, + int njoinrels, + int16 *joinleft, + int16 *joinright, + Oid *joinops, + int njoinquals, + int16 *stxkeys, + int16 *keyrefs, + int nkeys, + int stattarget, + int numrows, HeapTuple *rows, + double totalrows, + VacAttrStats ***result_stats); + extern MultiSortSupport multi_sort_init(int ndims); extern void multi_sort_add_dimension(MultiSortSupport mss, int sortdim, Oid oper, Oid collation); diff --git a/src/include/statistics/statistics.h b/src/include/statistics/statistics.h index 8f9b9d237fd..cb76fddfaff 100644 --- a/src/include/statistics/statistics.h +++ b/src/include/statistics/statistics.h @@ -128,4 +128,12 @@ extern StatisticExtInfo *choose_best_statistics(List *stats, char requiredkind, int nclauses); extern HeapTuple statext_expressions_load(Oid stxoid, bool inh, int idx); +/* Join MCV statistics functions */ +extern Selectivity join_mcv_clause_selectivity(PlannerInfo *root, + RestrictInfo *rinfo); +extern Selectivity statext_join_mcv_clauselist_selectivity(PlannerInfo *root, + List *clauses, + int varRelid, + Bitmapset **estimatedclauses); + #endif /* STATISTICS_H */ diff --git a/src/test/regress/expected/oidjoins.out b/src/test/regress/expected/oidjoins.out index d64169b7bf0..42550c0e1e2 100644 --- a/src/test/regress/expected/oidjoins.out +++ b/src/test/regress/expected/oidjoins.out @@ -24,6 +24,11 @@ begin end loop; cmd := cmd || ', unnest(' || quote_ident(fk.fkcols[nkeys]); cmd := cmd || ') as ' || quote_ident(fk.fkcols[nkeys]); + -- For pg_statistic_ext, include stxkind for join stats filtering + if fk.fktable = 'pg_statistic_ext'::regclass and + fk.fkcols = ARRAY['stxrelid', 'stxkeys'] then + cmd := cmd || ', stxkind, stxjoinrels'; + end if; cmd := cmd || ' FROM ' || fk.fktable::text || ') fk WHERE '; else cmd := cmd || ' FROM ' || fk.fktable::text || ' fk WHERE '; @@ -33,6 +38,13 @@ begin cmd := cmd || quote_ident(fk.fkcols[i]) || ' != 0 AND '; end loop; end if; + -- Special case: For join statistics, stxkeys references attributes from + -- the other table (via stxkeyrefs), not from stxrelid. Skip the FK + -- check for join stats where stxjoinrels is not null. + if fk.fktable = 'pg_statistic_ext'::regclass and + fk.fkcols = ARRAY['stxrelid', 'stxkeys'] then + cmd := cmd || 'stxjoinrels IS NULL AND '; + end if; cmd := cmd || 'NOT EXISTS(SELECT 1 FROM ' || fk.pktable::text || ' pk WHERE '; for i in 1 .. nkeys loop if i > 1 then cmd := cmd || ' AND '; end if; diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index a65a5bf0c4f..e689f0372b7 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -2704,9 +2704,14 @@ pg_stats_ext| SELECT cn.nspname AS schemaname, s.stxname AS statistics_name, s.oid AS statistics_id, pg_get_userbyid(s.stxowner) AS statistics_owner, - ( SELECT array_agg(a.attname ORDER BY a.attnum) AS array_agg - FROM (unnest(s.stxkeys) k(k) - JOIN pg_attribute a ON (((a.attrelid = s.stxrelid) AND (a.attnum = k.k))))) AS attnames, + ( SELECT array_agg(a.attname ORDER BY k.ord) AS array_agg + FROM ((unnest(s.stxkeys) WITH ORDINALITY k(attnum, ord) + LEFT JOIN unnest(s.stxkeyrefs) WITH ORDINALITY r(keyref, ord2) ON ((k.ord = r.ord2))) + JOIN pg_attribute a ON (((a.attrelid = + CASE + WHEN ((r.keyref IS NULL) OR (r.keyref = 1)) THEN s.stxrelid + ELSE s.stxjoinrels[(r.keyref - 2)] + END) AND (a.attnum = k.attnum))))) AS attnames, pg_get_statisticsobjdef_expressions(s.oid) AS exprs, s.stxkind AS kinds, sd.stxdinherit AS inherited, @@ -2726,7 +2731,10 @@ pg_stats_ext| SELECT cn.nspname AS schemaname, array_agg(pg_mcv_list_items.frequency) AS most_common_freqs, array_agg(pg_mcv_list_items.base_frequency) AS most_common_base_freqs FROM pg_mcv_list_items(sd.stxdmcv) pg_mcv_list_items(index, "values", nulls, frequency, base_frequency)) m ON ((sd.stxdmcv IS NOT NULL))) - WHERE (pg_has_role(c.relowner, 'USAGE'::text) AND ((c.relrowsecurity = false) OR (NOT row_security_active(c.oid)))); + WHERE (pg_has_role(c.relowner, 'USAGE'::text) AND ((s.stxjoinrels IS NULL) OR (NOT (EXISTS ( SELECT 1 + FROM (unnest(s.stxjoinrels) jr(oid) + JOIN pg_class jc ON ((jc.oid = jr.oid))) + WHERE (NOT pg_has_role(jc.relowner, 'USAGE'::text)))))) AND ((c.relrowsecurity = false) OR (NOT row_security_active(c.oid)))); pg_stats_ext_exprs| SELECT cn.nspname AS schemaname, c.relname AS tablename, s.stxrelid AS tableid, diff --git a/src/test/regress/expected/stats_ext.out b/src/test/regress/expected/stats_ext.out index 37070c1a896..ed034d9d7b0 100644 --- a/src/test/regress/expected/stats_ext.out +++ b/src/test/regress/expected/stats_ext.out @@ -56,22 +56,22 @@ CREATE STATISTICS tst (unrecognized) ON x, y FROM ext_stats_test; ERROR: unrecognized statistics kind "unrecognized" -- unsupported targets CREATE STATISTICS tst ON a FROM (VALUES (x)) AS foo; -ERROR: CREATE STATISTICS only supports relation names in the FROM clause +ERROR: CREATE STATISTICS only supports relation names or JOIN clauses in the FROM clause CREATE STATISTICS tst ON a FROM foo NATURAL JOIN bar; -ERROR: CREATE STATISTICS only supports relation names in the FROM clause +ERROR: relation "foo" does not exist CREATE STATISTICS tst ON a FROM (SELECT * FROM ext_stats_test) AS foo; -ERROR: CREATE STATISTICS only supports relation names in the FROM clause +ERROR: CREATE STATISTICS only supports relation names or JOIN clauses in the FROM clause CREATE STATISTICS tst ON a FROM ext_stats_test s TABLESAMPLE system (x); -ERROR: CREATE STATISTICS only supports relation names in the FROM clause +ERROR: CREATE STATISTICS only supports relation names or JOIN clauses in the FROM clause CREATE STATISTICS tst ON a FROM XMLTABLE('foo' PASSING 'bar' COLUMNS a text); -ERROR: CREATE STATISTICS only supports relation names in the FROM clause +ERROR: CREATE STATISTICS only supports relation names or JOIN clauses in the FROM clause CREATE STATISTICS tst ON a FROM JSON_TABLE(jsonb '123', '$' COLUMNS (item int)); -ERROR: CREATE STATISTICS only supports relation names in the FROM clause +ERROR: CREATE STATISTICS only supports relation names or JOIN clauses in the FROM clause CREATE FUNCTION tftest(int) returns table(a int, b int) as $$ SELECT $1, $1+i FROM generate_series(1,5) g(i); $$ LANGUAGE sql IMMUTABLE STRICT; CREATE STATISTICS alt_stat2 ON a FROM tftest(1); -ERROR: CREATE STATISTICS only supports relation names in the FROM clause +ERROR: CREATE STATISTICS only supports relation names or JOIN clauses in the FROM clause DROP FUNCTION tftest; -- incorrect expressions CREATE STATISTICS tst ON (y) FROM ext_stats_test; -- single column reference diff --git a/src/test/regress/expected/stats_ext_crossrel.out b/src/test/regress/expected/stats_ext_crossrel.out new file mode 100644 index 00000000000..7b945587527 --- /dev/null +++ b/src/test/regress/expected/stats_ext_crossrel.out @@ -0,0 +1,1312 @@ +-- Join MCV statistics tests +-- +-- Note: tables for which we check estimated row counts should be created +-- with autovacuum_enabled = off, so that we don't have unstable results +-- from auto-analyze happening when we didn't expect it. +-- +-- +-- Verify CREATE STATISTICS rejects invalid join syntax. +-- +-- Minimal tables for error-case testing. +CREATE TABLE t1 (id INTEGER PRIMARY KEY, val TEXT NOT NULL); +CREATE TABLE t2 (id INTEGER PRIMARY KEY, t1_id INTEGER NOT NULL); +INSERT INTO t1 VALUES (1, 'x'); +INSERT INTO t2 VALUES (1, 1); +ANALYZE t1; +ANALYZE t2; +CREATE STATISTICS bad_stats1 (mcv) ON t1.val; +ERROR: syntax error at or near ";" +LINE 1: CREATE STATISTICS bad_stats1 (mcv) ON t1.val; + ^ +CREATE STATISTICS bad_stats2 (mcv) ON t1.val FROM t2, t1; +ERROR: missing FROM-clause entry for table "t1" +LINE 1: CREATE STATISTICS bad_stats2 (mcv) ON t1.val FROM t2, t1; + ^ +CREATE STATISTICS bad_stats3 (mcv) FROM t2 JOIN t1 ON (t2.t1_id = t1.id); +ERROR: syntax error at or near "FROM" +LINE 1: CREATE STATISTICS bad_stats3 (mcv) FROM t2 JOIN t1 ON (t2.t1... + ^ +CREATE STATISTICS bad_stats4 (mcv) ON val FROM t2 JOIN t1 ON (t2.t1_id = t1.id); +ERROR: join statistics require table-qualified column names +CREATE STATISTICS bad_stats5 (mcv) ON lower(t1.val) FROM t2 JOIN t1 ON (t2.t1_id = t1.id); +ERROR: expressions are not supported in join statistics +-- Composite join condition (AND) is not supported +CREATE STATISTICS bad_composite (mcv) ON t1.val +FROM t2 JOIN t1 ON (t2.t1_id = t1.id AND t2.id = t1.id); +ERROR: join statistics require a single equijoin condition per pair of tables +-- Non-Var operand in join condition (constant) +CREATE STATISTICS bad_const_join (mcv) ON t1.val +FROM t2 JOIN t1 ON (t1.id = 1); +ERROR: join statistics require simple equijoin conditions +-- Non-equality join condition +CREATE STATISTICS bad_non_eq (mcv) ON t1.val +FROM t2 JOIN t1 ON (t2.t1_id > t1.id); +ERROR: join statistics require equijoin conditions +-- Non-inner join types are not supported +CREATE STATISTICS bad_left (mcv) ON t1.val +FROM t2 LEFT JOIN t1 ON (t2.t1_id = t1.id); +ERROR: join statistics are only supported for inner joins +CREATE STATISTICS bad_right (mcv) ON t1.val +FROM t2 RIGHT JOIN t1 ON (t2.t1_id = t1.id); +ERROR: join statistics are only supported for inner joins +CREATE STATISTICS bad_full (mcv) ON t1.val +FROM t2 FULL JOIN t1 ON (t2.t1_id = t1.id); +ERROR: join statistics are only supported for inner joins +CREATE STATISTICS bad_cross (mcv) ON t1.val +FROM t2 CROSS JOIN t1; +ERROR: join statistics require at least one join condition +HINT: Use JOIN ... ON instead of CROSS JOIN. +-- Subquery in FROM clause is not supported +CREATE STATISTICS bad_stats6 (mcv) ON x.val FROM (SELECT * FROM t1) x JOIN t2 ON (x.id = t2.t1_id); +ERROR: unsupported FROM clause element in join statistics +-- Self-join: both aliases refer to the same table. +-- CREATE succeeds; the stat is valid but rarely useful in practice. +CREATE STATISTICS self_join_stat (mcv) ON b.val +FROM t1 a JOIN t1 b ON (a.id = b.id); +DROP STATISTICS self_join_stat; +-- No suitable index on join column +CREATE TABLE no_idx (id INTEGER, val TEXT); +INSERT INTO no_idx VALUES (1, 'x'); +CREATE STATISTICS bad_stats7 (mcv) ON no_idx.val +FROM t2 JOIN no_idx ON (t2.t1_id = no_idx.id); +ERROR: no suitable index on "no_idx" column "id" for join statistics +HINT: Create an index on the join column to enable index-based join sampling. +-- BRIN index exists but doesn't support equality lookup +CREATE INDEX no_idx_brin ON no_idx USING brin (id); +CREATE STATISTICS bad_stats8 (mcv) ON no_idx.val +FROM t2 JOIN no_idx ON (t2.t1_id = no_idx.id); +ERROR: no suitable index on "no_idx" column "id" for join statistics +HINT: Create an index on the join column to enable index-based join sampling. +DROP INDEX no_idx_brin; +-- Partial index excluded (only indexes a subset of rows) +CREATE INDEX no_idx_partial ON no_idx (id) WHERE id > 0; +CREATE STATISTICS bad_stats_partial (mcv) ON no_idx.val +FROM t2 JOIN no_idx ON (t2.t1_id = no_idx.id); +ERROR: no suitable index on "no_idx" column "id" for join statistics +HINT: Create an index on the join column to enable index-based join sampling. +DROP INDEX no_idx_partial CASCADE; +-- Btree index supports equality lookup, succeeds +CREATE INDEX ON no_idx (id); +CREATE STATISTICS idx_dep_stats (mcv) ON no_idx.val +FROM t2 JOIN no_idx ON (t2.t1_id = no_idx.id); +-- DROP INDEX refuses due to dependency +DROP INDEX no_idx_id_idx; +ERROR: cannot drop index no_idx_id_idx because other objects depend on it +DETAIL: statistics object idx_dep_stats depends on index no_idx_id_idx +HINT: Use DROP ... CASCADE to drop the dependent objects too. +-- DROP INDEX CASCADE drops both the index and the dependent stats +DROP INDEX no_idx_id_idx CASCADE; +NOTICE: drop cascades to statistics object idx_dep_stats +SELECT count(*) FROM pg_statistic_ext WHERE stxname = 'idx_dep_stats'; + count +------- + 0 +(1 row) + +DROP TABLE no_idx; +-- 3-way join: CREATE STATISTICS succeeds, ANALYZE warns that collection +-- is not yet implemented for n-way joins. +CREATE TABLE t3 (id INTEGER PRIMARY KEY, label TEXT NOT NULL); +INSERT INTO t3 VALUES (1, 'a'); +ANALYZE t3; +-- 3-way join with filter columns from two different join relations. +-- Verify keyrefs maps each column to its correct source relation. +CREATE STATISTICS nway_multi_filter (mcv) ON t1.val, t3.label + FROM t2 + JOIN t1 ON (t2.t1_id = t1.id) + JOIN t3 ON (t1.id = t3.id); +-- keyrefs should be {2, 3}: val from varno 2=t1, +-- label from varno 3=t3 +SELECT s.stxname, + s.stxrelid::regclass, + s.stxjoinrels::regclass[], + s.stxkeyrefs, + s.stxkeys, + s.stxkind +FROM pg_statistic_ext s +WHERE s.stxname = 'nway_multi_filter'; + stxname | stxrelid | stxjoinrels | stxkeyrefs | stxkeys | stxkind +-------------------+----------+---------------+------------+---------+--------- + nway_multi_filter | t2 | [0:1]={t1,t3} | 2 3 | 2 2 | {m} +(1 row) + +SELECT pg_get_statisticsobjdef_columns_from(oid) FROM pg_statistic_ext WHERE stxname = 'nway_multi_filter'; + pg_get_statisticsobjdef_columns_from +-------------------------------------------------------------------------------------------- + t1.val, t3.label FROM t2 t2 JOIN t1 t1 ON (t2.t1_id = t1.id) JOIN t3 t3 ON (t1.id = t3.id) +(1 row) + +-- ANALYZE warns that n-way collection is not yet supported +ANALYZE t2; +WARNING: join statistics on more than two tables are not yet supported, skipping "nway_multi_filter" +DROP STATISTICS nway_multi_filter; +-- 3-way join with single filter: also warns +CREATE STATISTICS nway_stats (mcv) ON t1.val + FROM t2 + JOIN t1 ON (t2.t1_id = t1.id) + JOIN t3 ON (t1.id = t3.id); +ANALYZE t2; +WARNING: join statistics on more than two tables are not yet supported, skipping "nway_stats" +DROP STATISTICS nway_stats; +DROP TABLE t1, t2, t3; +-- +-- Test join MCV statistics creation, collection and usage. +-- +CREATE TABLE keywords ( + id INTEGER PRIMARY KEY, + keyword TEXT NOT NULL, + phonetic_code character varying(5) +) WITH (autovacuum_enabled = off); +CREATE TABLE movie_kw ( + movie_id INTEGER PRIMARY KEY, + keyword_id INTEGER NOT NULL, -- No FOREIGN KEY reference + year INTEGER NOT NULL +) WITH (autovacuum_enabled = off); +-- Insert tightly correlated data into the dimension table +INSERT INTO keywords (id, keyword, phonetic_code) +SELECT + i, + 'keyword_' || i, + CASE WHEN i BETWEEN 26 AND 30 THEN NULL ELSE 'ph_' || i END +FROM generate_series(1, 50) i; +-- Insert data into the fact table with skewed distribution. +-- year is correlated with keyword_id: ids 1-10 -> 2020, 11-20 -> 2021, +-- 21-30 -> 2022. +INSERT INTO movie_kw (movie_id, keyword_id, year) +SELECT + i, + CASE + WHEN i % 100 < 60 THEN (i % 10) + 1 -- 60% keyword_ids 1-10 (6% frequency per keyword) + WHEN i % 100 < 90 THEN (i % 10) + 11 -- 30% keyword_ids 11-20 (3% frequency per keyword) + ELSE (i % 10) + 21 -- 10% keyword_ids 21-30 (1% frequency per keyword) + END, + CASE + WHEN i % 100 < 60 THEN 2020 + WHEN i % 100 < 90 THEN 2021 + ELSE 2022 + END +FROM generate_series(1, 10000) i; +-- Create join MCV statistics on a single filter column (keyword). +CREATE STATISTICS jstats_one_col (mcv) +ON k.keyword +FROM movie_kw mk JOIN keywords k ON (mk.keyword_id = k.id); +-- Stats definition should be populated. +SELECT s.stxname, + s.stxrelid::regclass, + s.stxjoinrels::regclass[], + s.stxkeyrefs, + s.stxkeys, + s.stxkind +FROM pg_statistic_ext s +WHERE s.stxname = 'jstats_one_col'; + stxname | stxrelid | stxjoinrels | stxkeyrefs | stxkeys | stxkind +----------------+----------+------------------+------------+---------+--------- + jstats_one_col | movie_kw | [0:0]={keywords} | 2 | 2 | {m} +(1 row) + +SELECT pg_get_statisticsobjdef_columns_from(oid) FROM pg_statistic_ext WHERE stxname = 'jstats_one_col'; + pg_get_statisticsobjdef_columns_from +------------------------------------------------------------------------------------------------------- + keywords.keyword FROM movie_kw movie_kw JOIN keywords keywords ON (movie_kw.keyword_id = keywords.id) +(1 row) + +-- ANALYZE the joined table (non-anchor). +ANALYZE keywords; +-- The join stats data should not be collected yet. +SELECT d.stxdmcv IS NOT NULL AS has_mcv +FROM pg_statistic_ext s +LEFT JOIN pg_statistic_ext_data d ON (s.oid = d.stxoid) +WHERE s.stxname = 'jstats_one_col'; + has_mcv +--------- + f +(1 row) + +-- ANALYZE the anchor table +ANALYZE movie_kw; +-- Verify pg_stats_ext view resolves column names from the correct table. +-- jstats_one_col tracks keywords.keyword (attnum 2 in keywords, stxkeyrefs={2}). +-- The view must look up attnum 2 in keywords, not in movie_kw (the anchor). +SELECT attnames +FROM pg_stats_ext +WHERE statistics_name = 'jstats_one_col'; + attnames +----------- + {keyword} +(1 row) + +-- The join stats data should now be collected. +SELECT m.values, + m.nulls, + ROUND(m.frequency::numeric, 2) AS frequency +FROM pg_statistic_ext s +JOIN pg_statistic_ext_data d ON (s.oid = d.stxoid) +CROSS JOIN LATERAL pg_mcv_list_items(d.stxdmcv) AS m +WHERE s.stxname = 'jstats_one_col' +ORDER BY m.frequency DESC, m.values; + values | nulls | frequency +--------------+-------+----------- + {keyword_1} | {f} | 0.06 + {keyword_10} | {f} | 0.06 + {keyword_2} | {f} | 0.06 + {keyword_3} | {f} | 0.06 + {keyword_4} | {f} | 0.06 + {keyword_5} | {f} | 0.06 + {keyword_6} | {f} | 0.06 + {keyword_7} | {f} | 0.06 + {keyword_8} | {f} | 0.06 + {keyword_9} | {f} | 0.06 + {keyword_11} | {f} | 0.03 + {keyword_12} | {f} | 0.03 + {keyword_13} | {f} | 0.03 + {keyword_14} | {f} | 0.03 + {keyword_15} | {f} | 0.03 + {keyword_16} | {f} | 0.03 + {keyword_17} | {f} | 0.03 + {keyword_18} | {f} | 0.03 + {keyword_19} | {f} | 0.03 + {keyword_20} | {f} | 0.03 + {keyword_21} | {f} | 0.01 + {keyword_22} | {f} | 0.01 + {keyword_23} | {f} | 0.01 + {keyword_24} | {f} | 0.01 + {keyword_25} | {f} | 0.01 + {keyword_26} | {f} | 0.01 + {keyword_27} | {f} | 0.01 + {keyword_28} | {f} | 0.01 + {keyword_29} | {f} | 0.01 + {keyword_30} | {f} | 0.01 +(30 rows) + +-- Single equality filter on keyword. +SELECT * FROM check_estimated_rows(' + SELECT * FROM movie_kw mk, keywords k + WHERE k.keyword = ''keyword_1'' AND k.id = mk.keyword_id +'); + estimated | actual +-----------+-------- + 600 | 600 +(1 row) + +-- IN predicate on keyword. +SELECT * FROM check_estimated_rows(' + SELECT * FROM movie_kw mk, keywords k + WHERE k.keyword IN (''keyword_1'', ''keyword_2'', ''keyword_3'') + AND k.id = mk.keyword_id +'); + estimated | actual +-----------+-------- + 1800 | 1800 +(1 row) + +-- phonetic_code filter: not covered by the single-column keyword stat, +-- so the planner uses its default estimate. +SELECT * FROM check_estimated_rows(' + SELECT * FROM movie_kw mk, keywords k + WHERE k.phonetic_code = ''ph_1'' AND k.id = mk.keyword_id +'); + estimated | actual +-----------+-------- + 200 | 600 +(1 row) + +-- No filter on the joined table: the stat requires at least one covered +-- filter predicate, so the planner uses its default estimate. +SELECT * FROM check_estimated_rows(' + SELECT * FROM movie_kw mk, keywords k + WHERE k.id = mk.keyword_id +'); + estimated | actual +-----------+-------- + 10000 | 10000 +(1 row) + +-- Inequality join: the stat is built on an equality join, so it must not +-- apply when the join operator differs. +SELECT * FROM check_estimated_rows(' + SELECT * FROM movie_kw mk, keywords k + WHERE k.keyword = ''keyword_1'' AND mk.keyword_id > k.id +'); + estimated | actual +-----------+-------- + 3333 | 9400 +(1 row) + +-- Add a stat covering both keyword and phonetic_code. +CREATE STATISTICS jstats_two_col (mcv) +ON k.keyword, k.phonetic_code +FROM movie_kw mk JOIN keywords k ON (mk.keyword_id = k.id); +ANALYZE movie_kw; +-- Show the stats in catalog +SELECT s.stxname, + s.stxrelid::regclass, + s.stxjoinrels::regclass[], + s.stxkeyrefs, + s.stxkeys, + s.stxkind +FROM pg_statistic_ext s +WHERE s.stxname = 'jstats_two_col'; + stxname | stxrelid | stxjoinrels | stxkeyrefs | stxkeys | stxkind +----------------+----------+------------------+------------+---------+--------- + jstats_two_col | movie_kw | [0:0]={keywords} | 2 2 | 2 3 | {m} +(1 row) + +SELECT pg_get_statisticsobjdef_columns_from(oid) FROM pg_statistic_ext WHERE stxname = 'jstats_two_col'; + pg_get_statisticsobjdef_columns_from +------------------------------------------------------------------------------------------------------------------------------- + keywords.keyword, keywords.phonetic_code FROM movie_kw movie_kw JOIN keywords keywords ON (movie_kw.keyword_id = keywords.id) +(1 row) + +-- Verify pg_stats_ext resolves both columns from the joined table. +-- jstats_two_col tracks keywords.keyword and keywords.phonetic_code. +SELECT attnames +FROM pg_stats_ext +WHERE statistics_name = 'jstats_two_col'; + attnames +------------------------- + {keyword,phonetic_code} +(1 row) + +SELECT m.values, + m.nulls, + ROUND(m.frequency::numeric, 2) AS frequency +FROM pg_statistic_ext s +JOIN pg_statistic_ext_data d ON (s.oid = d.stxoid) +CROSS JOIN LATERAL pg_mcv_list_items(d.stxdmcv) AS m +WHERE s.stxname = 'jstats_two_col' +ORDER BY m.frequency DESC, m.values; + values | nulls | frequency +--------------------+-------+----------- + {keyword_1,ph_1} | {f,f} | 0.06 + {keyword_10,ph_10} | {f,f} | 0.06 + {keyword_2,ph_2} | {f,f} | 0.06 + {keyword_3,ph_3} | {f,f} | 0.06 + {keyword_4,ph_4} | {f,f} | 0.06 + {keyword_5,ph_5} | {f,f} | 0.06 + {keyword_6,ph_6} | {f,f} | 0.06 + {keyword_7,ph_7} | {f,f} | 0.06 + {keyword_8,ph_8} | {f,f} | 0.06 + {keyword_9,ph_9} | {f,f} | 0.06 + {keyword_11,ph_11} | {f,f} | 0.03 + {keyword_12,ph_12} | {f,f} | 0.03 + {keyword_13,ph_13} | {f,f} | 0.03 + {keyword_14,ph_14} | {f,f} | 0.03 + {keyword_15,ph_15} | {f,f} | 0.03 + {keyword_16,ph_16} | {f,f} | 0.03 + {keyword_17,ph_17} | {f,f} | 0.03 + {keyword_18,ph_18} | {f,f} | 0.03 + {keyword_19,ph_19} | {f,f} | 0.03 + {keyword_20,ph_20} | {f,f} | 0.03 + {keyword_21,ph_21} | {f,f} | 0.01 + {keyword_22,ph_22} | {f,f} | 0.01 + {keyword_23,ph_23} | {f,f} | 0.01 + {keyword_24,ph_24} | {f,f} | 0.01 + {keyword_25,ph_25} | {f,f} | 0.01 + {keyword_26,NULL} | {f,t} | 0.01 + {keyword_27,NULL} | {f,t} | 0.01 + {keyword_28,NULL} | {f,t} | 0.01 + {keyword_29,NULL} | {f,t} | 0.01 + {keyword_30,NULL} | {f,t} | 0.01 +(30 rows) + +-- phonetic_code filter: now covered by the multi-column stat, estimate improves. +SELECT * FROM check_estimated_rows(' + SELECT * FROM movie_kw mk, keywords k + WHERE k.phonetic_code = ''ph_1'' AND k.id = mk.keyword_id +'); + estimated | actual +-----------+-------- + 600 | 600 +(1 row) + +-- Both keyword and phonetic_code: fully covered by the multi-column stat. +SELECT * FROM check_estimated_rows(' + SELECT * FROM movie_kw mk, keywords k + WHERE k.keyword = ''keyword_1'' + AND k.phonetic_code = ''ph_1'' + AND k.id = mk.keyword_id +'); + estimated | actual +-----------+-------- + 600 | 600 +(1 row) + +-- Zero MCV match (keyword_1 never pairs with ph_15), falls back to +-- default estimation. +SELECT * FROM check_estimated_rows(' + SELECT * FROM movie_kw mk, keywords k + WHERE k.keyword = ''keyword_1'' + AND k.phonetic_code = ''ph_15'' + AND k.id = mk.keyword_id +'); + estimated | actual +-----------+-------- + 200 | 0 +(1 row) + +-- Filters from both tables: keyword from the joined table and year from the +-- anchor. keyword_1 always pairs with year=2020, but without a join stat +-- covering both tables the planner applies the two filters independently. +SELECT * FROM check_estimated_rows(' + SELECT * FROM movie_kw mk, keywords k + WHERE k.keyword = ''keyword_1'' AND mk.year = 2020 + AND k.id = mk.keyword_id +'); + estimated | actual +-----------+-------- + 120 | 600 +(1 row) + +-- Add a stat with filter columns from both tables. +CREATE STATISTICS jstats_two_tab (mcv) +ON k.keyword, mk.year +FROM movie_kw mk JOIN keywords k ON (mk.keyword_id = k.id); +ANALYZE movie_kw; +SELECT s.stxname, + s.stxrelid::regclass, + s.stxjoinrels::regclass[], + s.stxkeyrefs, + s.stxkeys, + s.stxkind +FROM pg_statistic_ext s +WHERE s.stxname = 'jstats_two_tab'; + stxname | stxrelid | stxjoinrels | stxkeyrefs | stxkeys | stxkind +----------------+----------+------------------+------------+---------+--------- + jstats_two_tab | movie_kw | [0:0]={keywords} | 2 1 | 2 3 | {m} +(1 row) + +SELECT pg_get_statisticsobjdef_columns_from(oid) FROM pg_statistic_ext WHERE stxname = 'jstats_two_tab'; + pg_get_statisticsobjdef_columns_from +---------------------------------------------------------------------------------------------------------------------- + keywords.keyword, movie_kw.year FROM movie_kw movie_kw JOIN keywords keywords ON (movie_kw.keyword_id = keywords.id) +(1 row) + +SELECT m.values, + m.nulls, + ROUND(m.frequency::numeric, 2) AS frequency +FROM pg_statistic_ext s +JOIN pg_statistic_ext_data d ON (s.oid = d.stxoid) +CROSS JOIN LATERAL pg_mcv_list_items(d.stxdmcv) AS m +WHERE s.stxname = 'jstats_two_tab' +ORDER BY m.frequency DESC, m.values; + values | nulls | frequency +-------------------+-------+----------- + {keyword_1,2020} | {f,f} | 0.06 + {keyword_10,2020} | {f,f} | 0.06 + {keyword_2,2020} | {f,f} | 0.06 + {keyword_3,2020} | {f,f} | 0.06 + {keyword_4,2020} | {f,f} | 0.06 + {keyword_5,2020} | {f,f} | 0.06 + {keyword_6,2020} | {f,f} | 0.06 + {keyword_7,2020} | {f,f} | 0.06 + {keyword_8,2020} | {f,f} | 0.06 + {keyword_9,2020} | {f,f} | 0.06 + {keyword_11,2021} | {f,f} | 0.03 + {keyword_12,2021} | {f,f} | 0.03 + {keyword_13,2021} | {f,f} | 0.03 + {keyword_14,2021} | {f,f} | 0.03 + {keyword_15,2021} | {f,f} | 0.03 + {keyword_16,2021} | {f,f} | 0.03 + {keyword_17,2021} | {f,f} | 0.03 + {keyword_18,2021} | {f,f} | 0.03 + {keyword_19,2021} | {f,f} | 0.03 + {keyword_20,2021} | {f,f} | 0.03 + {keyword_21,2022} | {f,f} | 0.01 + {keyword_22,2022} | {f,f} | 0.01 + {keyword_23,2022} | {f,f} | 0.01 + {keyword_24,2022} | {f,f} | 0.01 + {keyword_25,2022} | {f,f} | 0.01 + {keyword_26,2022} | {f,f} | 0.01 + {keyword_27,2022} | {f,f} | 0.01 + {keyword_28,2022} | {f,f} | 0.01 + {keyword_29,2022} | {f,f} | 0.01 + {keyword_30,2022} | {f,f} | 0.01 +(30 rows) + +-- Same query with both-table stat: estimate should improve. +SELECT * FROM check_estimated_rows(' + SELECT * FROM movie_kw mk, keywords k + WHERE k.keyword = ''keyword_1'' AND mk.year = 2020 + AND k.id = mk.keyword_id +'); + estimated | actual +-----------+-------- + 600 | 600 +(1 row) + +-- Contradictory filters from both tables: keyword_1 never appears with year 2021. +SELECT * FROM check_estimated_rows(' + SELECT * FROM movie_kw mk, keywords k + WHERE k.keyword = ''keyword_1'' AND mk.year = 2021 + AND k.id = mk.keyword_id +'); + estimated | actual +-----------+-------- + 60 | 0 +(1 row) + +-- Verify estimates hold when a foreign key constraint is present. +-- The planner estimates FK join selectivity through a separate code path; +-- these tests confirm that join MCV stats are still used. +ALTER TABLE movie_kw ADD CONSTRAINT mk_keyword_fk + FOREIGN KEY (keyword_id) REFERENCES keywords(id); +ANALYZE movie_kw; +-- Single filter through FK path. +SELECT * FROM check_estimated_rows(' + SELECT * FROM movie_kw mk, keywords k + WHERE k.keyword = ''keyword_1'' AND k.id = mk.keyword_id +'); + estimated | actual +-----------+-------- + 600 | 600 +(1 row) + +-- No filter: FK uses 1/ndistinct default. +SELECT * FROM check_estimated_rows(' + SELECT * FROM movie_kw mk, keywords k + WHERE k.id = mk.keyword_id +'); + estimated | actual +-----------+-------- + 10000 | 10000 +(1 row) + +-- Filters from both tables through FK path. +SELECT * FROM check_estimated_rows(' + SELECT * FROM movie_kw mk, keywords k + WHERE k.keyword = ''keyword_1'' AND mk.year = 2020 + AND k.id = mk.keyword_id +'); + estimated | actual +-----------+-------- + 600 | 600 +(1 row) + +-- Contradictory both-table filter through FK path: zero MCV match, +-- falls back to default estimation. +SELECT * FROM check_estimated_rows(' + SELECT * FROM movie_kw mk, keywords k + WHERE k.keyword = ''keyword_1'' AND mk.year = 2021 + AND k.id = mk.keyword_id +'); + estimated | actual +-----------+-------- + 60 | 0 +(1 row) + +ALTER TABLE movie_kw DROP CONSTRAINT mk_keyword_fk; +-- Verify \dX+ shows join statistics with full definition. +\dX+ jstats_one_col + List of extended statistics + Schema | Name | Definition | Ndistinct | Dependencies | MCV | Description +--------+----------------+-------------------------------------------------------------------------------------------------------+-----------+--------------+---------+------------- + public | jstats_one_col | keywords.keyword FROM movie_kw movie_kw JOIN keywords keywords ON (movie_kw.keyword_id = keywords.id) | | | defined | +(1 row) + +\dX+ jstats_two_tab + List of extended statistics + Schema | Name | Definition | Ndistinct | Dependencies | MCV | Description +--------+----------------+----------------------------------------------------------------------------------------------------------------------+-----------+--------------+---------+------------- + public | jstats_two_tab | keywords.keyword, movie_kw.year FROM movie_kw movie_kw JOIN keywords keywords ON (movie_kw.keyword_id = keywords.id) | | | defined | +(1 row) + +-- Verify \d+ on a table shows join statistics in the footer. +\d+ movie_kw + Table "public.movie_kw" + Column | Type | Collation | Nullable | Default | Storage | Stats target | Description +------------+---------+-----------+----------+---------+---------+--------------+------------- + movie_id | integer | | not null | | plain | | + keyword_id | integer | | not null | | plain | | + year | integer | | not null | | plain | | +Indexes: + "movie_kw_pkey" PRIMARY KEY, btree (movie_id) +Statistics objects: + "public.jstats_one_col" (mcv) ON keywords.keyword FROM movie_kw movie_kw JOIN keywords keywords ON (movie_kw.keyword_id = keywords.id) + "public.jstats_two_col" (mcv) ON keywords.keyword, keywords.phonetic_code FROM movie_kw movie_kw JOIN keywords keywords ON (movie_kw.keyword_id = keywords.id) + "public.jstats_two_tab" (mcv) ON keywords.keyword, movie_kw.year FROM movie_kw movie_kw JOIN keywords keywords ON (movie_kw.keyword_id = keywords.id) +Not-null constraints: + "movie_kw_movie_id_not_null" NOT NULL "movie_id" + "movie_kw_keyword_id_not_null" NOT NULL "keyword_id" + "movie_kw_year_not_null" NOT NULL "year" +Options: autovacuum_enabled=off + +DROP STATISTICS jstats_one_col; +DROP STATISTICS jstats_two_col; +DROP STATISTICS jstats_two_tab; +-- Reversed column declaration order: jstats_two_tab had {k.keyword, mk.year} +-- (keyrefs = {2, 1}), verify the same works with {mk.year, k.keyword} +-- (keyrefs = {1, 2}). +CREATE STATISTICS jstats_two_tab_rev (mcv) +ON mk.year, k.keyword +FROM movie_kw mk JOIN keywords k ON (mk.keyword_id = k.id); +ANALYZE movie_kw; +SELECT s.stxname, s.stxkeys, s.stxkeyrefs +FROM pg_statistic_ext s +WHERE s.stxname = 'jstats_two_tab_rev'; + stxname | stxkeys | stxkeyrefs +--------------------+---------+------------ + jstats_two_tab_rev | 3 2 | 1 2 +(1 row) + +SELECT * FROM check_estimated_rows(' + SELECT * FROM movie_kw mk, keywords k + WHERE k.keyword = ''keyword_1'' AND mk.year = 2020 + AND k.id = mk.keyword_id +'); + estimated | actual +-----------+-------- + 600 | 600 +(1 row) + +-- Verify keyrefs mapping is correct under FK path. +ALTER TABLE movie_kw ADD CONSTRAINT mk_keyword_fk + FOREIGN KEY (keyword_id) REFERENCES keywords(id); +ANALYZE movie_kw; +SELECT * FROM check_estimated_rows(' + SELECT * FROM movie_kw mk, keywords k + WHERE k.keyword = ''keyword_1'' AND mk.year = 2020 + AND k.id = mk.keyword_id +'); + estimated | actual +-----------+-------- + 600 | 600 +(1 row) + +ALTER TABLE movie_kw DROP CONSTRAINT mk_keyword_fk; +DROP STATISTICS jstats_two_tab_rev; +-- With a lower stattarget the join has more matches than the statistics +-- can keep, so ANALYZE must down-sample. Verify the result is still sane. +CREATE STATISTICS jstats_sampling (mcv) +ON k.keyword +FROM movie_kw mk JOIN keywords k ON (mk.keyword_id = k.id); +ALTER STATISTICS jstats_sampling SET STATISTICS 50; +ANALYZE movie_kw; +-- Verify MCV was built successfully via the sampling path. +SELECT d.stxdmcv IS NOT NULL AS has_mcv +FROM pg_statistic_ext s +JOIN pg_statistic_ext_data d ON (s.oid = d.stxoid) +WHERE s.stxname = 'jstats_sampling'; + has_mcv +--------- + t +(1 row) + +-- All 30 keywords with matches should be captured even with sampling. +SELECT COUNT(*) AS num_mcvs +FROM pg_statistic_ext s +JOIN pg_statistic_ext_data d ON (s.oid = d.stxoid) +CROSS JOIN LATERAL pg_mcv_list_items(d.stxdmcv) AS m +WHERE s.stxname = 'jstats_sampling'; + num_mcvs +---------- + 30 +(1 row) + +-- Frequency tiers should be preserved: 10 keywords at ~6%, 10 at ~3%, +-- 10 at ~1%. Use wide bins to absorb sampling variance. +SELECT + COUNT(*) FILTER (WHERE m.frequency >= 0.04) AS top_tier, + COUNT(*) FILTER (WHERE m.frequency >= 0.015 AND m.frequency < 0.04) AS mid_tier, + COUNT(*) FILTER (WHERE m.frequency > 0 AND m.frequency < 0.015) AS low_tier +FROM pg_statistic_ext s +JOIN pg_statistic_ext_data d ON (s.oid = d.stxoid) +CROSS JOIN LATERAL pg_mcv_list_items(d.stxdmcv) AS m +WHERE s.stxname = 'jstats_sampling'; + top_tier | mid_tier | low_tier +----------+----------+---------- + 10 | 10 | 10 +(1 row) + +DROP STATISTICS jstats_sampling; +-- Hash index: verify that a hash index is accepted and produces reasonable +-- MCV frequencies. +ALTER TABLE keywords DROP CONSTRAINT keywords_pkey; +CREATE INDEX keywords_hash ON keywords USING hash (id); +CREATE STATISTICS jstats_hash (mcv) +ON k.keyword +FROM movie_kw mk JOIN keywords k ON (mk.keyword_id = k.id); +ANALYZE movie_kw; +-- Verify MCV was built successfully. +SELECT d.stxdmcv IS NOT NULL AS has_mcv +FROM pg_statistic_ext s +JOIN pg_statistic_ext_data d ON (s.oid = d.stxoid) +WHERE s.stxname = 'jstats_hash'; + has_mcv +--------- + t +(1 row) + +-- All 30 keywords with matches should be captured. +SELECT COUNT(*) AS num_mcvs +FROM pg_statistic_ext s +JOIN pg_statistic_ext_data d ON (s.oid = d.stxoid) +CROSS JOIN LATERAL pg_mcv_list_items(d.stxdmcv) AS m +WHERE s.stxname = 'jstats_hash'; + num_mcvs +---------- + 30 +(1 row) + +-- Frequency tiers should match btree results: 10 at ~6%, 10 at ~3%, 10 at ~1%. +SELECT + COUNT(*) FILTER (WHERE m.frequency >= 0.04) AS top_tier, + COUNT(*) FILTER (WHERE m.frequency >= 0.015 AND m.frequency < 0.04) AS mid_tier, + COUNT(*) FILTER (WHERE m.frequency > 0 AND m.frequency < 0.015) AS low_tier +FROM pg_statistic_ext s +JOIN pg_statistic_ext_data d ON (s.oid = d.stxoid) +CROSS JOIN LATERAL pg_mcv_list_items(d.stxdmcv) AS m +WHERE s.stxname = 'jstats_hash'; + top_tier | mid_tier | low_tier +----------+----------+---------- + 10 | 10 | 10 +(1 row) + +-- Equality filter estimate should be accurate with hash index. +SELECT * FROM check_estimated_rows(' + SELECT * FROM movie_kw mk, keywords k + WHERE k.keyword = ''keyword_1'' AND k.id = mk.keyword_id +'); + estimated | actual +-----------+-------- + 600 | 600 +(1 row) + +DROP STATISTICS jstats_hash; +DROP INDEX keywords_hash; +-- Cleanup base tables +DROP TABLE movie_kw CASCADE; +DROP TABLE keywords CASCADE; +-- Partial filter coverage with single-table extended stats on the dimension. +-- +-- Setup: dimension table where color and shape are perfectly correlated +-- (red & circle, blue & square, etc.). A single-table extended stat captures +-- this correlation. The join stat covers only color, not shape. Queries +-- filter on both. This tests that partial coverage uses the join stat for +-- the covered filter and correctly accounts for the uncovered filter via +-- the single-table extended stat on other_rel. +CREATE TABLE dim_pc ( + id INTEGER PRIMARY KEY, + color TEXT NOT NULL, + shape TEXT NOT NULL +) WITH (autovacuum_enabled = off); +CREATE TABLE fact_pc ( + id INTEGER PRIMARY KEY, + dim_id INTEGER NOT NULL +) WITH (autovacuum_enabled = off); +-- 100 dim rows: 5 colors x 20 rows each, perfectly correlated with shape. +INSERT INTO dim_pc (id, color, shape) +SELECT i, + CASE (i - 1) / 20 + WHEN 0 THEN 'red' + WHEN 1 THEN 'blue' + WHEN 2 THEN 'green' + WHEN 3 THEN 'yellow' + ELSE 'white' + END, + CASE (i - 1) / 20 + WHEN 0 THEN 'circle' + WHEN 1 THEN 'square' + WHEN 2 THEN 'triangle' + WHEN 3 THEN 'star' + ELSE 'diamond' + END +FROM generate_series(1, 100) i; +-- 10000 fact rows with skewed distribution: +-- 50% -> dim ids 1-20 (red/circle), 30% -> 21-40 (blue/square), +-- 20% -> 41-60 (green/triangle), 0% -> 61-100. +INSERT INTO fact_pc (id, dim_id) +SELECT i, + CASE + WHEN i % 100 < 50 THEN (i % 20) + 1 + WHEN i % 100 < 80 THEN (i % 20) + 21 + ELSE (i % 20) + 41 + END +FROM generate_series(1, 10000) i; +ANALYZE dim_pc; +ANALYZE fact_pc; +-- Single-table extended stat on dim: captures color & shape correlation. +CREATE STATISTICS dim_pc_ext (mcv) ON color, shape FROM dim_pc; +ANALYZE dim_pc; +-- Verify _columns_from on a single-table stat returns columns + FROM. +SELECT pg_get_statisticsobjdef_columns_from(oid) FROM pg_statistic_ext WHERE stxname = 'dim_pc_ext'; + pg_get_statisticsobjdef_columns_from +-------------------------------------- + color, shape FROM dim_pc +(1 row) + +-- Baseline: without join stat, only single-table stats. +SELECT * FROM check_estimated_rows(' + SELECT * FROM fact_pc f, dim_pc d + WHERE d.color = ''red'' AND d.shape = ''circle'' + AND f.dim_id = d.id +'); + estimated | actual +-----------+-------- + 2000 | 5000 +(1 row) + +-- Baseline: contradictory filters, without join stat. +SELECT * FROM check_estimated_rows(' + SELECT * FROM fact_pc f, dim_pc d + WHERE d.color = ''red'' AND d.shape = ''square'' + AND f.dim_id = d.id +'); + estimated | actual +-----------+-------- + 100 | 0 +(1 row) + +-- Now add a join stat covering only color (not shape). +CREATE STATISTICS fact_dim_pc_color (mcv) ON d.color +FROM fact_pc f JOIN dim_pc d ON (f.dim_id = d.id); +ANALYZE fact_pc; +-- red & circle: consistent filters, 50% of fact rows = 5000 actual. +SELECT * FROM check_estimated_rows(' + SELECT * FROM fact_pc f, dim_pc d + WHERE d.color = ''red'' AND d.shape = ''circle'' + AND f.dim_id = d.id +'); + estimated | actual +-----------+-------- + 5000 | 5000 +(1 row) + +-- red & square: contradictory filters, 0 actual rows. +-- The single-table stat knows red never pairs with square. +SELECT * FROM check_estimated_rows(' + SELECT * FROM fact_pc f, dim_pc d + WHERE d.color = ''red'' AND d.shape = ''square'' + AND f.dim_id = d.id +'); + estimated | actual +-----------+-------- + 100 | 0 +(1 row) + +DROP TABLE fact_pc, dim_pc CASCADE; +-- Test that join stats are applied correctly in multi-way joins. +-- Create dimension tables for a star schema: facts joins dim1 and dim2. +CREATE TABLE facts ( + id INTEGER PRIMARY KEY, + dim1_id INTEGER NOT NULL, + dim2_id INTEGER NOT NULL +) WITH (autovacuum_enabled = off); +CREATE TABLE dim1 ( + id INTEGER PRIMARY KEY, + label TEXT NOT NULL +) WITH (autovacuum_enabled = off); +CREATE TABLE dim2 ( + id INTEGER PRIMARY KEY, + color TEXT NOT NULL +) WITH (autovacuum_enabled = off); +-- dim1: 50 rows +INSERT INTO dim1 (id, label) +SELECT i, 'label_' || i FROM generate_series(1, 50) i; +-- dim2: 20 rows +INSERT INTO dim2 (id, color) +SELECT i, 'color_' || i FROM generate_series(1, 20) i; +-- facts: 10000 rows with skewed distributions to both dimensions. +-- dim1_id: 60% go to ids 1-10 (6% each), 30% to 11-20, 10% to 21-30. +-- dim2_id: 50% go to id 1, 30% to id 2, 20% spread across 3-20. +INSERT INTO facts (id, dim1_id, dim2_id) +SELECT i, + CASE + WHEN i % 100 < 60 THEN (i % 10) + 1 + WHEN i % 100 < 90 THEN (i % 10) + 11 + ELSE (i % 10) + 21 + END, + CASE + WHEN i % 100 < 50 THEN 1 + WHEN i % 100 < 80 THEN 2 + ELSE (i % 20) + 3 + END +FROM generate_series(1, 10000) i; +ANALYZE facts; +ANALYZE dim1; +ANALYZE dim2; +-- Create join stats on both pairs: facts-dim1 and facts-dim2. +CREATE STATISTICS facts_dim1_stat (mcv) ON d1.label +FROM facts f JOIN dim1 d1 ON (f.dim1_id = d1.id); +ANALYZE facts; +CREATE STATISTICS facts_dim2_stat (mcv) ON d2.color +FROM facts f JOIN dim2 d2 ON (f.dim2_id = d2.id); +ANALYZE facts; +-- Test 1: 2-way baselines. These should use the join stats directly. +-- label_1 -> dim1.id=1 -> 6% of facts = 600 rows. +-- Without stat the planner estimates 10000/50 = 200. +SELECT * FROM check_estimated_rows(' + SELECT * FROM facts f JOIN dim1 d1 ON (f.dim1_id = d1.id) + WHERE d1.label = ''label_1'' +'); + estimated | actual +-----------+-------- + 600 | 600 +(1 row) + +-- color_1 -> dim2.id=1 -> 50% of facts = 5000 rows. +-- Without stat the planner estimates 10000/20 = 500. +SELECT * FROM check_estimated_rows(' + SELECT * FROM facts f JOIN dim2 d2 ON (f.dim2_id = d2.id) + WHERE d2.color = ''color_1'' +'); + estimated | actual +-----------+-------- + 5102 | 5000 +(1 row) + +-- Test 2: 3-way join with ONE stat applicable. +-- facts joins dim1 and a plain table (no stat). The facts-dim1 stat +-- should still be found even when one join side is a join rel. +SELECT * FROM check_estimated_rows(' + SELECT * FROM facts f + JOIN dim2 d2 ON (f.dim2_id = d2.id) + JOIN dim1 d1 ON (f.dim1_id = d1.id) + WHERE d1.label = ''label_1'' +'); + estimated | actual +-----------+-------- + 545 | 600 +(1 row) + +-- Test 3: 3-way join with BOTH stats applicable. +-- facts joins dim1 (filter label_1) and dim2 (filter color_1). +-- Both stats should contribute independently: 10000 * 6% * 50% = 300. +SELECT * FROM check_estimated_rows(' + SELECT * FROM facts f + JOIN dim1 d1 ON (f.dim1_id = d1.id) + JOIN dim2 d2 ON (f.dim2_id = d2.id) + WHERE d1.label = ''label_1'' AND d2.color = ''color_1'' +'); + estimated | actual +-----------+-------- + 306 | 500 +(1 row) + +-- Test 4: Same 3-way query, different FROM order. The estimate should +-- be the same regardless of how the tables are listed. +SELECT * FROM check_estimated_rows(' + SELECT * FROM dim2 d2 + JOIN facts f ON (f.dim2_id = d2.id) + JOIN dim1 d1 ON (f.dim1_id = d1.id) + WHERE d1.label = ''label_1'' AND d2.color = ''color_1'' +'); + estimated | actual +-----------+-------- + 306 | 500 +(1 row) + +-- Test 5: Verify that dropping one stat degrades only that dimension. +-- Drop facts-dim2 stat; the dim2 estimate should fall back to default +-- while dim1 still uses the stat. +DROP STATISTICS facts_dim2_stat; +SELECT * FROM check_estimated_rows(' + SELECT * FROM facts f + JOIN dim1 d1 ON (f.dim1_id = d1.id) + JOIN dim2 d2 ON (f.dim2_id = d2.id) + WHERE d1.label = ''label_1'' AND d2.color = ''color_1'' +'); + estimated | actual +-----------+-------- + 27 | 500 +(1 row) + +-- Test 6: No stats at all. Verify the partial-stat estimate (test 5) is +-- not worse than the no-stat baseline. +DROP STATISTICS facts_dim1_stat; +SELECT * FROM check_estimated_rows(' + SELECT * FROM facts f + JOIN dim1 d1 ON (f.dim1_id = d1.id) + JOIN dim2 d2 ON (f.dim2_id = d2.id) + WHERE d1.label = ''label_1'' AND d2.color = ''color_1'' +'); + estimated | actual +-----------+-------- + 9 | 500 +(1 row) + +DROP TABLE facts, dim1, dim2; +-- Multi-column FK: join stats should not alter the FK selectivity. +-- +-- A composite FK (fk_a, fk_b) REFERENCES dim(a, b) produces two join +-- clauses. Join stats currently only support single-condition joins, so +-- they cannot cover both FK columns. The FK path should skip the stat +-- entirely. +CREATE TABLE dim_mfk ( + a INTEGER NOT NULL, + b INTEGER NOT NULL, + label TEXT NOT NULL, + PRIMARY KEY (a, b) +) WITH (autovacuum_enabled = off); +CREATE TABLE fact_mfk ( + id INTEGER PRIMARY KEY, + fk_a INTEGER NOT NULL, + fk_b INTEGER NOT NULL, + FOREIGN KEY (fk_a, fk_b) REFERENCES dim_mfk(a, b) +) WITH (autovacuum_enabled = off); +-- dim: 50 rows, a in 1..10, b in 1..5, label skewed by a. +INSERT INTO dim_mfk (a, b, label) +SELECT (i - 1) / 5 + 1, (i - 1) % 5 + 1, + CASE WHEN (i - 1) / 5 + 1 <= 3 THEN 'hot' ELSE 'cold' END +FROM generate_series(1, 50) i; +-- fact: 10000 rows, fk_a skewed toward low values, fk_b uniform. +INSERT INTO fact_mfk (id, fk_a, fk_b) +SELECT i, + CASE + WHEN i % 100 < 60 THEN (i % 3) + 1 + WHEN i % 100 < 90 THEN (i % 3) + 4 + ELSE (i % 4) + 7 + END, + (i % 5) + 1 +FROM generate_series(1, 10000) i; +ANALYZE dim_mfk; +ANALYZE fact_mfk; +-- Baseline: default FK selectivity, no join stat. +SELECT * FROM check_estimated_rows(' + SELECT * FROM fact_mfk f, dim_mfk d + WHERE d.label = ''hot'' AND f.fk_a = d.a AND f.fk_b = d.b +'); + estimated | actual +-----------+-------- + 3000 | 6000 +(1 row) + +-- Add join stat covering only one FK column (fk_a = dim.a). +CREATE STATISTICS fact_dim_mfk (mcv) ON d.label +FROM fact_mfk f JOIN dim_mfk d ON (f.fk_a = d.a); +ANALYZE fact_mfk; +-- With join stat: the estimate must stay the same as the baseline. +-- The stat covers only one of the two FK columns, so the FK path should +-- ignore it and fall back to the default 1/ref_tuples. +SELECT * FROM check_estimated_rows(' + SELECT * FROM fact_mfk f, dim_mfk d + WHERE d.label = ''hot'' AND f.fk_a = d.a AND f.fk_b = d.b +'); + estimated | actual +-----------+-------- + 3000 | 6000 +(1 row) + +DROP TABLE fact_mfk, dim_mfk CASCADE; +-- +-- Non-MCV value estimation -- single-key case. +-- Estimates should not be worse than without join stats. +-- +CREATE TABLE tail_dim ( + id INTEGER PRIMARY KEY, + label TEXT NOT NULL +) WITH (autovacuum_enabled = off); +CREATE TABLE tail_fact ( + dim_id INTEGER NOT NULL +) WITH (autovacuum_enabled = off); +INSERT INTO tail_dim VALUES + (1, 'common_a'), (2, 'common_b'), + (3, 'other_a'), (4, 'other_b'), + (5, 'extra_1'), (6, 'extra_2'); -- dimension-only +INSERT INTO tail_fact (dim_id) +SELECT 1 FROM generate_series(1, 90) -- common_a +UNION ALL SELECT 2 FROM generate_series(1, 70) -- common_b +UNION ALL SELECT 3 FROM generate_series(1, 25) -- other_a +UNION ALL SELECT 4 FROM generate_series(1, 15); -- other_b +ANALYZE tail_dim; +ANALYZE tail_fact; +-- Baselines without join stat. +SELECT * FROM check_estimated_rows(' + SELECT * FROM tail_fact f, tail_dim d + WHERE d.label IN (''common_a'', ''other_a'') AND d.id = f.dim_id +'); + estimated | actual +-----------+-------- + 67 | 115 +(1 row) + +SELECT * FROM check_estimated_rows(' + SELECT * FROM tail_fact f, tail_dim d + WHERE d.label = ''other_a'' AND d.id = f.dim_id +'); + estimated | actual +-----------+-------- + 33 | 25 +(1 row) + +-- stattarget=2: only the top 2 items land in the MCV. +CREATE STATISTICS jstats_tail (mcv) ON d.label +FROM tail_fact f JOIN tail_dim d ON (f.dim_id = d.id); +ALTER STATISTICS jstats_tail SET STATISTICS 2; +ANALYZE tail_fact; +-- Test 1: IN-list mixing MCV + non-MCV value (same query as baseline). +SELECT * FROM check_estimated_rows(' + SELECT * FROM tail_fact f, tail_dim d + WHERE d.label IN (''common_a'', ''other_a'') AND d.id = f.dim_id +'); + estimated | actual +-----------+-------- + 100 | 115 +(1 row) + +-- Test 2: single non-MCV value (same query as baseline). +SELECT * FROM check_estimated_rows(' + SELECT * FROM tail_fact f, tail_dim d + WHERE d.label = ''other_a'' AND d.id = f.dim_id +'); + estimated | actual +-----------+-------- + 33 | 25 +(1 row) + +DROP STATISTICS jstats_tail; +DROP TABLE tail_fact, tail_dim; +-- +-- Non-MCV value estimation -- multi-key case. +-- Estimates should not be worse than without join stats. +-- +CREATE TABLE tail_dim2 ( + id INTEGER PRIMARY KEY, + color TEXT NOT NULL +) WITH (autovacuum_enabled = off); +CREATE TABLE tail_fact2 ( + dim_id INTEGER NOT NULL, + category TEXT NOT NULL +) WITH (autovacuum_enabled = off); +INSERT INTO tail_dim2 VALUES + (1, 'common'), (2, 'other'), + (3, 'extra_1'), (4, 'extra_2'); -- dimension-only +-- 4 combinations = 2 categories x 2 colors; top 2 are both on 'common'. +INSERT INTO tail_fact2 (dim_id, category) +SELECT 1, 'A' FROM generate_series(1, 60) -- (common, A) +UNION ALL SELECT 1, 'B' FROM generate_series(1, 55) -- (common, B) +UNION ALL SELECT 2, 'A' FROM generate_series(1, 43) -- (other, A) +UNION ALL SELECT 2, 'B' FROM generate_series(1, 42); -- (other, B) +ANALYZE tail_dim2; +ANALYZE tail_fact2; +-- Baselines without join stat. +SELECT * FROM check_estimated_rows(' + SELECT * FROM tail_fact2 f, tail_dim2 d + WHERE d.color IN (''common'', ''other'') AND d.id = f.dim_id +'); + estimated | actual +-----------+-------- + 100 | 200 +(1 row) + +SELECT * FROM check_estimated_rows(' + SELECT * FROM tail_fact2 f, tail_dim2 d + WHERE d.color = ''other'' AND d.id = f.dim_id +'); + estimated | actual +-----------+-------- + 50 | 85 +(1 row) + +-- stattarget=2: only (A,common) and (B,common) land in the MCV. +CREATE STATISTICS jstats_tail2 (mcv) ON f.category, d.color +FROM tail_fact2 f JOIN tail_dim2 d ON (f.dim_id = d.id); +ALTER STATISTICS jstats_tail2 SET STATISTICS 2; +ANALYZE tail_fact2; +-- Test 1: IN-list mixing MCV + non-MCV color (same query as baseline). +SELECT * FROM check_estimated_rows(' + SELECT * FROM tail_fact2 f, tail_dim2 d + WHERE d.color IN (''common'', ''other'') AND d.id = f.dim_id +'); + estimated | actual +-----------+-------- + 143 | 200 +(1 row) + +-- Test 2: single non-MCV value (same query as baseline). +SELECT * FROM check_estimated_rows(' + SELECT * FROM tail_fact2 f, tail_dim2 d + WHERE d.color = ''other'' AND d.id = f.dim_id +'); + estimated | actual +-----------+-------- + 50 | 85 +(1 row) + +DROP STATISTICS jstats_tail2; +DROP TABLE tail_fact2, tail_dim2; +-- +-- Cross-type join: float4 anchor probing a float8 index. +-- The join uses the float48eq cross-type operator in btree/float_ops. +-- Data is skewed: 80% of fact rows reference 'red' dimension rows. +-- +CREATE TABLE ct_dim (id float8 PRIMARY KEY, color text NOT NULL) + WITH (autovacuum_enabled = off); +CREATE TABLE ct_fact (id serial, dim_id float4 NOT NULL) + WITH (autovacuum_enabled = off); +-- 15 dimension rows: ids 1-5 are 'red', 6-10 'blue', 11-15 'green'. +INSERT INTO ct_dim SELECT i::float8, CASE WHEN i <= 5 THEN 'red' + WHEN i <= 10 THEN 'blue' + ELSE 'green' END +FROM generate_series(1, 15) i; +-- 500 fact rows: 80% reference red (ids 1-5), 20% reference others. +-- Standard estimator assumes uniform and underestimates the red share. +INSERT INTO ct_fact (dim_id) +SELECT CASE WHEN i % 5 < 4 THEN (i % 5 + 1)::float4 + ELSE (6 + i % 10)::float4 END +FROM generate_series(1, 500) i; +ANALYZE ct_dim; +ANALYZE ct_fact; +-- Baseline: without join stat, standard estimator assumes uniform. +SELECT * FROM check_estimated_rows(' + SELECT * FROM ct_fact f, ct_dim d + WHERE f.dim_id = d.id AND d.color = ''red'' +'); + estimated | actual +-----------+-------- + 167 | 400 +(1 row) + +CREATE STATISTICS jstats_crosstype (mcv) ON ct_fact.dim_id, ct_dim.color +FROM ct_fact JOIN ct_dim ON (ct_fact.dim_id = ct_dim.id); +ANALYZE ct_fact; +-- The stat should be populated (stxdmcv IS NOT NULL). +SELECT stxname FROM pg_statistic_ext +WHERE stxname = 'jstats_crosstype' +AND EXISTS ( + SELECT 1 FROM pg_statistic_ext_data d + WHERE d.stxoid = pg_statistic_ext.oid + AND d.stxdmcv IS NOT NULL +); + stxname +------------------ + jstats_crosstype +(1 row) + +-- With join stat, estimate should improve for the skewed data. +SELECT * FROM check_estimated_rows(' + SELECT * FROM ct_fact f, ct_dim d + WHERE f.dim_id = d.id AND d.color = ''red'' +'); + estimated | actual +-----------+-------- + 400 | 400 +(1 row) + +DROP STATISTICS jstats_crosstype; +DROP TABLE ct_fact, ct_dim; +-- +-- Same-named tables in different schemas: deparse must disambiguate aliases. +-- +CREATE SCHEMA m5_s1; +CREATE SCHEMA m5_s2; +CREATE TABLE m5_s1.t (a int PRIMARY KEY) WITH (autovacuum_enabled = off); +CREATE TABLE m5_s2.t (b int) WITH (autovacuum_enabled = off); +CREATE INDEX ON m5_s2.t (b); +INSERT INTO m5_s1.t SELECT generate_series(1, 100); +INSERT INTO m5_s2.t SELECT generate_series(1, 100); +ANALYZE m5_s1.t; +ANALYZE m5_s2.t; +CREATE STATISTICS m5_jstats (mcv) ON m5_s2.t.b +FROM m5_s1.t JOIN m5_s2.t ON (m5_s1.t.a = m5_s2.t.b); +-- The deparsed DDL must use distinct aliases for identically-named tables. +SELECT pg_get_statisticsobjdef(oid) FROM pg_statistic_ext +WHERE stxname = 'm5_jstats'; + pg_get_statisticsobjdef +---------------------------------------------------------------------------------------------------- + CREATE STATISTICS public.m5_jstats (mcv) ON t_1.b FROM m5_s1.t t JOIN m5_s2.t t_1 ON (t.a = t_1.b) +(1 row) + +DROP STATISTICS m5_jstats; +DROP TABLE m5_s1.t, m5_s2.t; +DROP SCHEMA m5_s1; +DROP SCHEMA m5_s2; diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index 5d4f910155e..329a3b9f0e7 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -83,6 +83,9 @@ test: create_table_like alter_generic alter_operator misc async dbsize merge mis # amutils depends on geometry, create_index_spgist, hash_index, brin test: rules psql psql_crosstab psql_pipeline amutils stats_ext collate.linux.utf8 collate.windows.win1252 +# join MCV statistics tests +test: stats_ext_crossrel + # ---------- # Run these alone so they don't run out of parallel workers # select_parallel depends on create_misc diff --git a/src/test/regress/sql/oidjoins.sql b/src/test/regress/sql/oidjoins.sql index 8b22e6d10c5..2b08ae7b24c 100644 --- a/src/test/regress/sql/oidjoins.sql +++ b/src/test/regress/sql/oidjoins.sql @@ -24,6 +24,11 @@ begin end loop; cmd := cmd || ', unnest(' || quote_ident(fk.fkcols[nkeys]); cmd := cmd || ') as ' || quote_ident(fk.fkcols[nkeys]); + -- For pg_statistic_ext, include stxkind for join stats filtering + if fk.fktable = 'pg_statistic_ext'::regclass and + fk.fkcols = ARRAY['stxrelid', 'stxkeys'] then + cmd := cmd || ', stxkind, stxjoinrels'; + end if; cmd := cmd || ' FROM ' || fk.fktable::text || ') fk WHERE '; else cmd := cmd || ' FROM ' || fk.fktable::text || ' fk WHERE '; @@ -33,6 +38,13 @@ begin cmd := cmd || quote_ident(fk.fkcols[i]) || ' != 0 AND '; end loop; end if; + -- Special case: For join statistics, stxkeys references attributes from + -- the other table (via stxkeyrefs), not from stxrelid. Skip the FK + -- check for join stats where stxjoinrels is not null. + if fk.fktable = 'pg_statistic_ext'::regclass and + fk.fkcols = ARRAY['stxrelid', 'stxkeys'] then + cmd := cmd || 'stxjoinrels IS NULL AND '; + end if; cmd := cmd || 'NOT EXISTS(SELECT 1 FROM ' || fk.pktable::text || ' pk WHERE '; for i in 1 .. nkeys loop if i > 1 then cmd := cmd || ' AND '; end if; diff --git a/src/test/regress/sql/stats_ext_crossrel.sql b/src/test/regress/sql/stats_ext_crossrel.sql new file mode 100644 index 00000000000..080ad049dce --- /dev/null +++ b/src/test/regress/sql/stats_ext_crossrel.sql @@ -0,0 +1,991 @@ +-- Join MCV statistics tests +-- +-- Note: tables for which we check estimated row counts should be created +-- with autovacuum_enabled = off, so that we don't have unstable results +-- from auto-analyze happening when we didn't expect it. +-- + +-- +-- Verify CREATE STATISTICS rejects invalid join syntax. +-- + +-- Minimal tables for error-case testing. +CREATE TABLE t1 (id INTEGER PRIMARY KEY, val TEXT NOT NULL); +CREATE TABLE t2 (id INTEGER PRIMARY KEY, t1_id INTEGER NOT NULL); +INSERT INTO t1 VALUES (1, 'x'); +INSERT INTO t2 VALUES (1, 1); +ANALYZE t1; +ANALYZE t2; + +CREATE STATISTICS bad_stats1 (mcv) ON t1.val; +CREATE STATISTICS bad_stats2 (mcv) ON t1.val FROM t2, t1; +CREATE STATISTICS bad_stats3 (mcv) FROM t2 JOIN t1 ON (t2.t1_id = t1.id); +CREATE STATISTICS bad_stats4 (mcv) ON val FROM t2 JOIN t1 ON (t2.t1_id = t1.id); +CREATE STATISTICS bad_stats5 (mcv) ON lower(t1.val) FROM t2 JOIN t1 ON (t2.t1_id = t1.id); +-- Composite join condition (AND) is not supported +CREATE STATISTICS bad_composite (mcv) ON t1.val +FROM t2 JOIN t1 ON (t2.t1_id = t1.id AND t2.id = t1.id); +-- Non-Var operand in join condition (constant) +CREATE STATISTICS bad_const_join (mcv) ON t1.val +FROM t2 JOIN t1 ON (t1.id = 1); +-- Non-equality join condition +CREATE STATISTICS bad_non_eq (mcv) ON t1.val +FROM t2 JOIN t1 ON (t2.t1_id > t1.id); +-- Non-inner join types are not supported +CREATE STATISTICS bad_left (mcv) ON t1.val +FROM t2 LEFT JOIN t1 ON (t2.t1_id = t1.id); +CREATE STATISTICS bad_right (mcv) ON t1.val +FROM t2 RIGHT JOIN t1 ON (t2.t1_id = t1.id); +CREATE STATISTICS bad_full (mcv) ON t1.val +FROM t2 FULL JOIN t1 ON (t2.t1_id = t1.id); +CREATE STATISTICS bad_cross (mcv) ON t1.val +FROM t2 CROSS JOIN t1; +-- Subquery in FROM clause is not supported +CREATE STATISTICS bad_stats6 (mcv) ON x.val FROM (SELECT * FROM t1) x JOIN t2 ON (x.id = t2.t1_id); +-- Self-join: both aliases refer to the same table. +-- CREATE succeeds; the stat is valid but rarely useful in practice. +CREATE STATISTICS self_join_stat (mcv) ON b.val +FROM t1 a JOIN t1 b ON (a.id = b.id); +DROP STATISTICS self_join_stat; +-- No suitable index on join column +CREATE TABLE no_idx (id INTEGER, val TEXT); +INSERT INTO no_idx VALUES (1, 'x'); +CREATE STATISTICS bad_stats7 (mcv) ON no_idx.val +FROM t2 JOIN no_idx ON (t2.t1_id = no_idx.id); +-- BRIN index exists but doesn't support equality lookup +CREATE INDEX no_idx_brin ON no_idx USING brin (id); +CREATE STATISTICS bad_stats8 (mcv) ON no_idx.val +FROM t2 JOIN no_idx ON (t2.t1_id = no_idx.id); +DROP INDEX no_idx_brin; +-- Partial index excluded (only indexes a subset of rows) +CREATE INDEX no_idx_partial ON no_idx (id) WHERE id > 0; +CREATE STATISTICS bad_stats_partial (mcv) ON no_idx.val +FROM t2 JOIN no_idx ON (t2.t1_id = no_idx.id); +DROP INDEX no_idx_partial CASCADE; +-- Btree index supports equality lookup, succeeds +CREATE INDEX ON no_idx (id); +CREATE STATISTICS idx_dep_stats (mcv) ON no_idx.val +FROM t2 JOIN no_idx ON (t2.t1_id = no_idx.id); +-- DROP INDEX refuses due to dependency +DROP INDEX no_idx_id_idx; +-- DROP INDEX CASCADE drops both the index and the dependent stats +DROP INDEX no_idx_id_idx CASCADE; +SELECT count(*) FROM pg_statistic_ext WHERE stxname = 'idx_dep_stats'; +DROP TABLE no_idx; + +-- 3-way join: CREATE STATISTICS succeeds, ANALYZE warns that collection +-- is not yet implemented for n-way joins. +CREATE TABLE t3 (id INTEGER PRIMARY KEY, label TEXT NOT NULL); +INSERT INTO t3 VALUES (1, 'a'); +ANALYZE t3; + +-- 3-way join with filter columns from two different join relations. +-- Verify keyrefs maps each column to its correct source relation. +CREATE STATISTICS nway_multi_filter (mcv) ON t1.val, t3.label + FROM t2 + JOIN t1 ON (t2.t1_id = t1.id) + JOIN t3 ON (t1.id = t3.id); + +-- keyrefs should be {2, 3}: val from varno 2=t1, +-- label from varno 3=t3 +SELECT s.stxname, + s.stxrelid::regclass, + s.stxjoinrels::regclass[], + s.stxkeyrefs, + s.stxkeys, + s.stxkind +FROM pg_statistic_ext s +WHERE s.stxname = 'nway_multi_filter'; + +SELECT pg_get_statisticsobjdef_columns_from(oid) FROM pg_statistic_ext WHERE stxname = 'nway_multi_filter'; + +-- ANALYZE warns that n-way collection is not yet supported +ANALYZE t2; + +DROP STATISTICS nway_multi_filter; + +-- 3-way join with single filter: also warns +CREATE STATISTICS nway_stats (mcv) ON t1.val + FROM t2 + JOIN t1 ON (t2.t1_id = t1.id) + JOIN t3 ON (t1.id = t3.id); + +ANALYZE t2; + +DROP STATISTICS nway_stats; +DROP TABLE t1, t2, t3; + +-- +-- Test join MCV statistics creation, collection and usage. +-- + +CREATE TABLE keywords ( + id INTEGER PRIMARY KEY, + keyword TEXT NOT NULL, + phonetic_code character varying(5) +) WITH (autovacuum_enabled = off); + +CREATE TABLE movie_kw ( + movie_id INTEGER PRIMARY KEY, + keyword_id INTEGER NOT NULL, -- No FOREIGN KEY reference + year INTEGER NOT NULL +) WITH (autovacuum_enabled = off); + +-- Insert tightly correlated data into the dimension table +INSERT INTO keywords (id, keyword, phonetic_code) +SELECT + i, + 'keyword_' || i, + CASE WHEN i BETWEEN 26 AND 30 THEN NULL ELSE 'ph_' || i END +FROM generate_series(1, 50) i; + +-- Insert data into the fact table with skewed distribution. +-- year is correlated with keyword_id: ids 1-10 -> 2020, 11-20 -> 2021, +-- 21-30 -> 2022. +INSERT INTO movie_kw (movie_id, keyword_id, year) +SELECT + i, + CASE + WHEN i % 100 < 60 THEN (i % 10) + 1 -- 60% keyword_ids 1-10 (6% frequency per keyword) + WHEN i % 100 < 90 THEN (i % 10) + 11 -- 30% keyword_ids 11-20 (3% frequency per keyword) + ELSE (i % 10) + 21 -- 10% keyword_ids 21-30 (1% frequency per keyword) + END, + CASE + WHEN i % 100 < 60 THEN 2020 + WHEN i % 100 < 90 THEN 2021 + ELSE 2022 + END +FROM generate_series(1, 10000) i; + +-- Create join MCV statistics on a single filter column (keyword). +CREATE STATISTICS jstats_one_col (mcv) +ON k.keyword +FROM movie_kw mk JOIN keywords k ON (mk.keyword_id = k.id); + +-- Stats definition should be populated. +SELECT s.stxname, + s.stxrelid::regclass, + s.stxjoinrels::regclass[], + s.stxkeyrefs, + s.stxkeys, + s.stxkind +FROM pg_statistic_ext s +WHERE s.stxname = 'jstats_one_col'; + +SELECT pg_get_statisticsobjdef_columns_from(oid) FROM pg_statistic_ext WHERE stxname = 'jstats_one_col'; + +-- ANALYZE the joined table (non-anchor). +ANALYZE keywords; + +-- The join stats data should not be collected yet. +SELECT d.stxdmcv IS NOT NULL AS has_mcv +FROM pg_statistic_ext s +LEFT JOIN pg_statistic_ext_data d ON (s.oid = d.stxoid) +WHERE s.stxname = 'jstats_one_col'; + +-- ANALYZE the anchor table +ANALYZE movie_kw; + +-- Verify pg_stats_ext view resolves column names from the correct table. +-- jstats_one_col tracks keywords.keyword (attnum 2 in keywords, stxkeyrefs={2}). +-- The view must look up attnum 2 in keywords, not in movie_kw (the anchor). +SELECT attnames +FROM pg_stats_ext +WHERE statistics_name = 'jstats_one_col'; + +-- The join stats data should now be collected. +SELECT m.values, + m.nulls, + ROUND(m.frequency::numeric, 2) AS frequency +FROM pg_statistic_ext s +JOIN pg_statistic_ext_data d ON (s.oid = d.stxoid) +CROSS JOIN LATERAL pg_mcv_list_items(d.stxdmcv) AS m +WHERE s.stxname = 'jstats_one_col' +ORDER BY m.frequency DESC, m.values; + +-- Single equality filter on keyword. +SELECT * FROM check_estimated_rows(' + SELECT * FROM movie_kw mk, keywords k + WHERE k.keyword = ''keyword_1'' AND k.id = mk.keyword_id +'); + +-- IN predicate on keyword. +SELECT * FROM check_estimated_rows(' + SELECT * FROM movie_kw mk, keywords k + WHERE k.keyword IN (''keyword_1'', ''keyword_2'', ''keyword_3'') + AND k.id = mk.keyword_id +'); + +-- phonetic_code filter: not covered by the single-column keyword stat, +-- so the planner uses its default estimate. +SELECT * FROM check_estimated_rows(' + SELECT * FROM movie_kw mk, keywords k + WHERE k.phonetic_code = ''ph_1'' AND k.id = mk.keyword_id +'); + +-- No filter on the joined table: the stat requires at least one covered +-- filter predicate, so the planner uses its default estimate. +SELECT * FROM check_estimated_rows(' + SELECT * FROM movie_kw mk, keywords k + WHERE k.id = mk.keyword_id +'); + +-- Inequality join: the stat is built on an equality join, so it must not +-- apply when the join operator differs. +SELECT * FROM check_estimated_rows(' + SELECT * FROM movie_kw mk, keywords k + WHERE k.keyword = ''keyword_1'' AND mk.keyword_id > k.id +'); + +-- Add a stat covering both keyword and phonetic_code. +CREATE STATISTICS jstats_two_col (mcv) +ON k.keyword, k.phonetic_code +FROM movie_kw mk JOIN keywords k ON (mk.keyword_id = k.id); +ANALYZE movie_kw; + +-- Show the stats in catalog +SELECT s.stxname, + s.stxrelid::regclass, + s.stxjoinrels::regclass[], + s.stxkeyrefs, + s.stxkeys, + s.stxkind +FROM pg_statistic_ext s +WHERE s.stxname = 'jstats_two_col'; + +SELECT pg_get_statisticsobjdef_columns_from(oid) FROM pg_statistic_ext WHERE stxname = 'jstats_two_col'; + +-- Verify pg_stats_ext resolves both columns from the joined table. +-- jstats_two_col tracks keywords.keyword and keywords.phonetic_code. +SELECT attnames +FROM pg_stats_ext +WHERE statistics_name = 'jstats_two_col'; + +SELECT m.values, + m.nulls, + ROUND(m.frequency::numeric, 2) AS frequency +FROM pg_statistic_ext s +JOIN pg_statistic_ext_data d ON (s.oid = d.stxoid) +CROSS JOIN LATERAL pg_mcv_list_items(d.stxdmcv) AS m +WHERE s.stxname = 'jstats_two_col' +ORDER BY m.frequency DESC, m.values; + +-- phonetic_code filter: now covered by the multi-column stat, estimate improves. +SELECT * FROM check_estimated_rows(' + SELECT * FROM movie_kw mk, keywords k + WHERE k.phonetic_code = ''ph_1'' AND k.id = mk.keyword_id +'); + +-- Both keyword and phonetic_code: fully covered by the multi-column stat. +SELECT * FROM check_estimated_rows(' + SELECT * FROM movie_kw mk, keywords k + WHERE k.keyword = ''keyword_1'' + AND k.phonetic_code = ''ph_1'' + AND k.id = mk.keyword_id +'); + +-- Zero MCV match (keyword_1 never pairs with ph_15), falls back to +-- default estimation. +SELECT * FROM check_estimated_rows(' + SELECT * FROM movie_kw mk, keywords k + WHERE k.keyword = ''keyword_1'' + AND k.phonetic_code = ''ph_15'' + AND k.id = mk.keyword_id +'); + +-- Filters from both tables: keyword from the joined table and year from the +-- anchor. keyword_1 always pairs with year=2020, but without a join stat +-- covering both tables the planner applies the two filters independently. +SELECT * FROM check_estimated_rows(' + SELECT * FROM movie_kw mk, keywords k + WHERE k.keyword = ''keyword_1'' AND mk.year = 2020 + AND k.id = mk.keyword_id +'); + +-- Add a stat with filter columns from both tables. +CREATE STATISTICS jstats_two_tab (mcv) +ON k.keyword, mk.year +FROM movie_kw mk JOIN keywords k ON (mk.keyword_id = k.id); +ANALYZE movie_kw; + +SELECT s.stxname, + s.stxrelid::regclass, + s.stxjoinrels::regclass[], + s.stxkeyrefs, + s.stxkeys, + s.stxkind +FROM pg_statistic_ext s +WHERE s.stxname = 'jstats_two_tab'; + +SELECT pg_get_statisticsobjdef_columns_from(oid) FROM pg_statistic_ext WHERE stxname = 'jstats_two_tab'; + +SELECT m.values, + m.nulls, + ROUND(m.frequency::numeric, 2) AS frequency +FROM pg_statistic_ext s +JOIN pg_statistic_ext_data d ON (s.oid = d.stxoid) +CROSS JOIN LATERAL pg_mcv_list_items(d.stxdmcv) AS m +WHERE s.stxname = 'jstats_two_tab' +ORDER BY m.frequency DESC, m.values; + +-- Same query with both-table stat: estimate should improve. +SELECT * FROM check_estimated_rows(' + SELECT * FROM movie_kw mk, keywords k + WHERE k.keyword = ''keyword_1'' AND mk.year = 2020 + AND k.id = mk.keyword_id +'); + +-- Contradictory filters from both tables: keyword_1 never appears with year 2021. +SELECT * FROM check_estimated_rows(' + SELECT * FROM movie_kw mk, keywords k + WHERE k.keyword = ''keyword_1'' AND mk.year = 2021 + AND k.id = mk.keyword_id +'); + +-- Verify estimates hold when a foreign key constraint is present. +-- The planner estimates FK join selectivity through a separate code path; +-- these tests confirm that join MCV stats are still used. +ALTER TABLE movie_kw ADD CONSTRAINT mk_keyword_fk + FOREIGN KEY (keyword_id) REFERENCES keywords(id); +ANALYZE movie_kw; + +-- Single filter through FK path. +SELECT * FROM check_estimated_rows(' + SELECT * FROM movie_kw mk, keywords k + WHERE k.keyword = ''keyword_1'' AND k.id = mk.keyword_id +'); + +-- No filter: FK uses 1/ndistinct default. +SELECT * FROM check_estimated_rows(' + SELECT * FROM movie_kw mk, keywords k + WHERE k.id = mk.keyword_id +'); + +-- Filters from both tables through FK path. +SELECT * FROM check_estimated_rows(' + SELECT * FROM movie_kw mk, keywords k + WHERE k.keyword = ''keyword_1'' AND mk.year = 2020 + AND k.id = mk.keyword_id +'); + +-- Contradictory both-table filter through FK path: zero MCV match, +-- falls back to default estimation. +SELECT * FROM check_estimated_rows(' + SELECT * FROM movie_kw mk, keywords k + WHERE k.keyword = ''keyword_1'' AND mk.year = 2021 + AND k.id = mk.keyword_id +'); + +ALTER TABLE movie_kw DROP CONSTRAINT mk_keyword_fk; + +-- Verify \dX+ shows join statistics with full definition. +\dX+ jstats_one_col +\dX+ jstats_two_tab + +-- Verify \d+ on a table shows join statistics in the footer. +\d+ movie_kw + +DROP STATISTICS jstats_one_col; +DROP STATISTICS jstats_two_col; +DROP STATISTICS jstats_two_tab; + +-- Reversed column declaration order: jstats_two_tab had {k.keyword, mk.year} +-- (keyrefs = {2, 1}), verify the same works with {mk.year, k.keyword} +-- (keyrefs = {1, 2}). +CREATE STATISTICS jstats_two_tab_rev (mcv) +ON mk.year, k.keyword +FROM movie_kw mk JOIN keywords k ON (mk.keyword_id = k.id); +ANALYZE movie_kw; + +SELECT s.stxname, s.stxkeys, s.stxkeyrefs +FROM pg_statistic_ext s +WHERE s.stxname = 'jstats_two_tab_rev'; + +SELECT * FROM check_estimated_rows(' + SELECT * FROM movie_kw mk, keywords k + WHERE k.keyword = ''keyword_1'' AND mk.year = 2020 + AND k.id = mk.keyword_id +'); + +-- Verify keyrefs mapping is correct under FK path. +ALTER TABLE movie_kw ADD CONSTRAINT mk_keyword_fk + FOREIGN KEY (keyword_id) REFERENCES keywords(id); +ANALYZE movie_kw; + +SELECT * FROM check_estimated_rows(' + SELECT * FROM movie_kw mk, keywords k + WHERE k.keyword = ''keyword_1'' AND mk.year = 2020 + AND k.id = mk.keyword_id +'); + +ALTER TABLE movie_kw DROP CONSTRAINT mk_keyword_fk; +DROP STATISTICS jstats_two_tab_rev; + +-- With a lower stattarget the join has more matches than the statistics +-- can keep, so ANALYZE must down-sample. Verify the result is still sane. +CREATE STATISTICS jstats_sampling (mcv) +ON k.keyword +FROM movie_kw mk JOIN keywords k ON (mk.keyword_id = k.id); +ALTER STATISTICS jstats_sampling SET STATISTICS 50; +ANALYZE movie_kw; + +-- Verify MCV was built successfully via the sampling path. +SELECT d.stxdmcv IS NOT NULL AS has_mcv +FROM pg_statistic_ext s +JOIN pg_statistic_ext_data d ON (s.oid = d.stxoid) +WHERE s.stxname = 'jstats_sampling'; + +-- All 30 keywords with matches should be captured even with sampling. +SELECT COUNT(*) AS num_mcvs +FROM pg_statistic_ext s +JOIN pg_statistic_ext_data d ON (s.oid = d.stxoid) +CROSS JOIN LATERAL pg_mcv_list_items(d.stxdmcv) AS m +WHERE s.stxname = 'jstats_sampling'; + +-- Frequency tiers should be preserved: 10 keywords at ~6%, 10 at ~3%, +-- 10 at ~1%. Use wide bins to absorb sampling variance. +SELECT + COUNT(*) FILTER (WHERE m.frequency >= 0.04) AS top_tier, + COUNT(*) FILTER (WHERE m.frequency >= 0.015 AND m.frequency < 0.04) AS mid_tier, + COUNT(*) FILTER (WHERE m.frequency > 0 AND m.frequency < 0.015) AS low_tier +FROM pg_statistic_ext s +JOIN pg_statistic_ext_data d ON (s.oid = d.stxoid) +CROSS JOIN LATERAL pg_mcv_list_items(d.stxdmcv) AS m +WHERE s.stxname = 'jstats_sampling'; + +DROP STATISTICS jstats_sampling; + +-- Hash index: verify that a hash index is accepted and produces reasonable +-- MCV frequencies. +ALTER TABLE keywords DROP CONSTRAINT keywords_pkey; +CREATE INDEX keywords_hash ON keywords USING hash (id); +CREATE STATISTICS jstats_hash (mcv) +ON k.keyword +FROM movie_kw mk JOIN keywords k ON (mk.keyword_id = k.id); +ANALYZE movie_kw; + +-- Verify MCV was built successfully. +SELECT d.stxdmcv IS NOT NULL AS has_mcv +FROM pg_statistic_ext s +JOIN pg_statistic_ext_data d ON (s.oid = d.stxoid) +WHERE s.stxname = 'jstats_hash'; + +-- All 30 keywords with matches should be captured. +SELECT COUNT(*) AS num_mcvs +FROM pg_statistic_ext s +JOIN pg_statistic_ext_data d ON (s.oid = d.stxoid) +CROSS JOIN LATERAL pg_mcv_list_items(d.stxdmcv) AS m +WHERE s.stxname = 'jstats_hash'; + +-- Frequency tiers should match btree results: 10 at ~6%, 10 at ~3%, 10 at ~1%. +SELECT + COUNT(*) FILTER (WHERE m.frequency >= 0.04) AS top_tier, + COUNT(*) FILTER (WHERE m.frequency >= 0.015 AND m.frequency < 0.04) AS mid_tier, + COUNT(*) FILTER (WHERE m.frequency > 0 AND m.frequency < 0.015) AS low_tier +FROM pg_statistic_ext s +JOIN pg_statistic_ext_data d ON (s.oid = d.stxoid) +CROSS JOIN LATERAL pg_mcv_list_items(d.stxdmcv) AS m +WHERE s.stxname = 'jstats_hash'; + +-- Equality filter estimate should be accurate with hash index. +SELECT * FROM check_estimated_rows(' + SELECT * FROM movie_kw mk, keywords k + WHERE k.keyword = ''keyword_1'' AND k.id = mk.keyword_id +'); + +DROP STATISTICS jstats_hash; +DROP INDEX keywords_hash; + +-- Cleanup base tables +DROP TABLE movie_kw CASCADE; +DROP TABLE keywords CASCADE; + +-- Partial filter coverage with single-table extended stats on the dimension. +-- +-- Setup: dimension table where color and shape are perfectly correlated +-- (red & circle, blue & square, etc.). A single-table extended stat captures +-- this correlation. The join stat covers only color, not shape. Queries +-- filter on both. This tests that partial coverage uses the join stat for +-- the covered filter and correctly accounts for the uncovered filter via +-- the single-table extended stat on other_rel. +CREATE TABLE dim_pc ( + id INTEGER PRIMARY KEY, + color TEXT NOT NULL, + shape TEXT NOT NULL +) WITH (autovacuum_enabled = off); + +CREATE TABLE fact_pc ( + id INTEGER PRIMARY KEY, + dim_id INTEGER NOT NULL +) WITH (autovacuum_enabled = off); + +-- 100 dim rows: 5 colors x 20 rows each, perfectly correlated with shape. +INSERT INTO dim_pc (id, color, shape) +SELECT i, + CASE (i - 1) / 20 + WHEN 0 THEN 'red' + WHEN 1 THEN 'blue' + WHEN 2 THEN 'green' + WHEN 3 THEN 'yellow' + ELSE 'white' + END, + CASE (i - 1) / 20 + WHEN 0 THEN 'circle' + WHEN 1 THEN 'square' + WHEN 2 THEN 'triangle' + WHEN 3 THEN 'star' + ELSE 'diamond' + END +FROM generate_series(1, 100) i; + +-- 10000 fact rows with skewed distribution: +-- 50% -> dim ids 1-20 (red/circle), 30% -> 21-40 (blue/square), +-- 20% -> 41-60 (green/triangle), 0% -> 61-100. +INSERT INTO fact_pc (id, dim_id) +SELECT i, + CASE + WHEN i % 100 < 50 THEN (i % 20) + 1 + WHEN i % 100 < 80 THEN (i % 20) + 21 + ELSE (i % 20) + 41 + END +FROM generate_series(1, 10000) i; + +ANALYZE dim_pc; +ANALYZE fact_pc; + +-- Single-table extended stat on dim: captures color & shape correlation. +CREATE STATISTICS dim_pc_ext (mcv) ON color, shape FROM dim_pc; +ANALYZE dim_pc; + +-- Verify _columns_from on a single-table stat returns columns + FROM. +SELECT pg_get_statisticsobjdef_columns_from(oid) FROM pg_statistic_ext WHERE stxname = 'dim_pc_ext'; + +-- Baseline: without join stat, only single-table stats. +SELECT * FROM check_estimated_rows(' + SELECT * FROM fact_pc f, dim_pc d + WHERE d.color = ''red'' AND d.shape = ''circle'' + AND f.dim_id = d.id +'); + +-- Baseline: contradictory filters, without join stat. +SELECT * FROM check_estimated_rows(' + SELECT * FROM fact_pc f, dim_pc d + WHERE d.color = ''red'' AND d.shape = ''square'' + AND f.dim_id = d.id +'); + +-- Now add a join stat covering only color (not shape). +CREATE STATISTICS fact_dim_pc_color (mcv) ON d.color +FROM fact_pc f JOIN dim_pc d ON (f.dim_id = d.id); +ANALYZE fact_pc; + +-- red & circle: consistent filters, 50% of fact rows = 5000 actual. +SELECT * FROM check_estimated_rows(' + SELECT * FROM fact_pc f, dim_pc d + WHERE d.color = ''red'' AND d.shape = ''circle'' + AND f.dim_id = d.id +'); + +-- red & square: contradictory filters, 0 actual rows. +-- The single-table stat knows red never pairs with square. +SELECT * FROM check_estimated_rows(' + SELECT * FROM fact_pc f, dim_pc d + WHERE d.color = ''red'' AND d.shape = ''square'' + AND f.dim_id = d.id +'); + +DROP TABLE fact_pc, dim_pc CASCADE; + +-- Test that join stats are applied correctly in multi-way joins. + +-- Create dimension tables for a star schema: facts joins dim1 and dim2. +CREATE TABLE facts ( + id INTEGER PRIMARY KEY, + dim1_id INTEGER NOT NULL, + dim2_id INTEGER NOT NULL +) WITH (autovacuum_enabled = off); + +CREATE TABLE dim1 ( + id INTEGER PRIMARY KEY, + label TEXT NOT NULL +) WITH (autovacuum_enabled = off); + +CREATE TABLE dim2 ( + id INTEGER PRIMARY KEY, + color TEXT NOT NULL +) WITH (autovacuum_enabled = off); + +-- dim1: 50 rows +INSERT INTO dim1 (id, label) +SELECT i, 'label_' || i FROM generate_series(1, 50) i; + +-- dim2: 20 rows +INSERT INTO dim2 (id, color) +SELECT i, 'color_' || i FROM generate_series(1, 20) i; + +-- facts: 10000 rows with skewed distributions to both dimensions. +-- dim1_id: 60% go to ids 1-10 (6% each), 30% to 11-20, 10% to 21-30. +-- dim2_id: 50% go to id 1, 30% to id 2, 20% spread across 3-20. +INSERT INTO facts (id, dim1_id, dim2_id) +SELECT i, + CASE + WHEN i % 100 < 60 THEN (i % 10) + 1 + WHEN i % 100 < 90 THEN (i % 10) + 11 + ELSE (i % 10) + 21 + END, + CASE + WHEN i % 100 < 50 THEN 1 + WHEN i % 100 < 80 THEN 2 + ELSE (i % 20) + 3 + END +FROM generate_series(1, 10000) i; + +ANALYZE facts; +ANALYZE dim1; +ANALYZE dim2; + + +-- Create join stats on both pairs: facts-dim1 and facts-dim2. +CREATE STATISTICS facts_dim1_stat (mcv) ON d1.label +FROM facts f JOIN dim1 d1 ON (f.dim1_id = d1.id); +ANALYZE facts; + +CREATE STATISTICS facts_dim2_stat (mcv) ON d2.color +FROM facts f JOIN dim2 d2 ON (f.dim2_id = d2.id); +ANALYZE facts; + +-- Test 1: 2-way baselines. These should use the join stats directly. +-- label_1 -> dim1.id=1 -> 6% of facts = 600 rows. +-- Without stat the planner estimates 10000/50 = 200. +SELECT * FROM check_estimated_rows(' + SELECT * FROM facts f JOIN dim1 d1 ON (f.dim1_id = d1.id) + WHERE d1.label = ''label_1'' +'); + +-- color_1 -> dim2.id=1 -> 50% of facts = 5000 rows. +-- Without stat the planner estimates 10000/20 = 500. +SELECT * FROM check_estimated_rows(' + SELECT * FROM facts f JOIN dim2 d2 ON (f.dim2_id = d2.id) + WHERE d2.color = ''color_1'' +'); + +-- Test 2: 3-way join with ONE stat applicable. +-- facts joins dim1 and a plain table (no stat). The facts-dim1 stat +-- should still be found even when one join side is a join rel. +SELECT * FROM check_estimated_rows(' + SELECT * FROM facts f + JOIN dim2 d2 ON (f.dim2_id = d2.id) + JOIN dim1 d1 ON (f.dim1_id = d1.id) + WHERE d1.label = ''label_1'' +'); + +-- Test 3: 3-way join with BOTH stats applicable. +-- facts joins dim1 (filter label_1) and dim2 (filter color_1). +-- Both stats should contribute independently: 10000 * 6% * 50% = 300. +SELECT * FROM check_estimated_rows(' + SELECT * FROM facts f + JOIN dim1 d1 ON (f.dim1_id = d1.id) + JOIN dim2 d2 ON (f.dim2_id = d2.id) + WHERE d1.label = ''label_1'' AND d2.color = ''color_1'' +'); + +-- Test 4: Same 3-way query, different FROM order. The estimate should +-- be the same regardless of how the tables are listed. +SELECT * FROM check_estimated_rows(' + SELECT * FROM dim2 d2 + JOIN facts f ON (f.dim2_id = d2.id) + JOIN dim1 d1 ON (f.dim1_id = d1.id) + WHERE d1.label = ''label_1'' AND d2.color = ''color_1'' +'); + +-- Test 5: Verify that dropping one stat degrades only that dimension. +-- Drop facts-dim2 stat; the dim2 estimate should fall back to default +-- while dim1 still uses the stat. +DROP STATISTICS facts_dim2_stat; + +SELECT * FROM check_estimated_rows(' + SELECT * FROM facts f + JOIN dim1 d1 ON (f.dim1_id = d1.id) + JOIN dim2 d2 ON (f.dim2_id = d2.id) + WHERE d1.label = ''label_1'' AND d2.color = ''color_1'' +'); + +-- Test 6: No stats at all. Verify the partial-stat estimate (test 5) is +-- not worse than the no-stat baseline. +DROP STATISTICS facts_dim1_stat; + +SELECT * FROM check_estimated_rows(' + SELECT * FROM facts f + JOIN dim1 d1 ON (f.dim1_id = d1.id) + JOIN dim2 d2 ON (f.dim2_id = d2.id) + WHERE d1.label = ''label_1'' AND d2.color = ''color_1'' +'); + +DROP TABLE facts, dim1, dim2; + +-- Multi-column FK: join stats should not alter the FK selectivity. +-- +-- A composite FK (fk_a, fk_b) REFERENCES dim(a, b) produces two join +-- clauses. Join stats currently only support single-condition joins, so +-- they cannot cover both FK columns. The FK path should skip the stat +-- entirely. + +CREATE TABLE dim_mfk ( + a INTEGER NOT NULL, + b INTEGER NOT NULL, + label TEXT NOT NULL, + PRIMARY KEY (a, b) +) WITH (autovacuum_enabled = off); + +CREATE TABLE fact_mfk ( + id INTEGER PRIMARY KEY, + fk_a INTEGER NOT NULL, + fk_b INTEGER NOT NULL, + FOREIGN KEY (fk_a, fk_b) REFERENCES dim_mfk(a, b) +) WITH (autovacuum_enabled = off); + +-- dim: 50 rows, a in 1..10, b in 1..5, label skewed by a. +INSERT INTO dim_mfk (a, b, label) +SELECT (i - 1) / 5 + 1, (i - 1) % 5 + 1, + CASE WHEN (i - 1) / 5 + 1 <= 3 THEN 'hot' ELSE 'cold' END +FROM generate_series(1, 50) i; + +-- fact: 10000 rows, fk_a skewed toward low values, fk_b uniform. +INSERT INTO fact_mfk (id, fk_a, fk_b) +SELECT i, + CASE + WHEN i % 100 < 60 THEN (i % 3) + 1 + WHEN i % 100 < 90 THEN (i % 3) + 4 + ELSE (i % 4) + 7 + END, + (i % 5) + 1 +FROM generate_series(1, 10000) i; + +ANALYZE dim_mfk; +ANALYZE fact_mfk; + +-- Baseline: default FK selectivity, no join stat. +SELECT * FROM check_estimated_rows(' + SELECT * FROM fact_mfk f, dim_mfk d + WHERE d.label = ''hot'' AND f.fk_a = d.a AND f.fk_b = d.b +'); + +-- Add join stat covering only one FK column (fk_a = dim.a). +CREATE STATISTICS fact_dim_mfk (mcv) ON d.label +FROM fact_mfk f JOIN dim_mfk d ON (f.fk_a = d.a); +ANALYZE fact_mfk; + +-- With join stat: the estimate must stay the same as the baseline. +-- The stat covers only one of the two FK columns, so the FK path should +-- ignore it and fall back to the default 1/ref_tuples. +SELECT * FROM check_estimated_rows(' + SELECT * FROM fact_mfk f, dim_mfk d + WHERE d.label = ''hot'' AND f.fk_a = d.a AND f.fk_b = d.b +'); + +DROP TABLE fact_mfk, dim_mfk CASCADE; + +-- +-- Non-MCV value estimation -- single-key case. +-- Estimates should not be worse than without join stats. +-- + +CREATE TABLE tail_dim ( + id INTEGER PRIMARY KEY, + label TEXT NOT NULL +) WITH (autovacuum_enabled = off); + +CREATE TABLE tail_fact ( + dim_id INTEGER NOT NULL +) WITH (autovacuum_enabled = off); + +INSERT INTO tail_dim VALUES + (1, 'common_a'), (2, 'common_b'), + (3, 'other_a'), (4, 'other_b'), + (5, 'extra_1'), (6, 'extra_2'); -- dimension-only + +INSERT INTO tail_fact (dim_id) +SELECT 1 FROM generate_series(1, 90) -- common_a +UNION ALL SELECT 2 FROM generate_series(1, 70) -- common_b +UNION ALL SELECT 3 FROM generate_series(1, 25) -- other_a +UNION ALL SELECT 4 FROM generate_series(1, 15); -- other_b + +ANALYZE tail_dim; +ANALYZE tail_fact; + +-- Baselines without join stat. +SELECT * FROM check_estimated_rows(' + SELECT * FROM tail_fact f, tail_dim d + WHERE d.label IN (''common_a'', ''other_a'') AND d.id = f.dim_id +'); + +SELECT * FROM check_estimated_rows(' + SELECT * FROM tail_fact f, tail_dim d + WHERE d.label = ''other_a'' AND d.id = f.dim_id +'); + +-- stattarget=2: only the top 2 items land in the MCV. +CREATE STATISTICS jstats_tail (mcv) ON d.label +FROM tail_fact f JOIN tail_dim d ON (f.dim_id = d.id); +ALTER STATISTICS jstats_tail SET STATISTICS 2; +ANALYZE tail_fact; + +-- Test 1: IN-list mixing MCV + non-MCV value (same query as baseline). +SELECT * FROM check_estimated_rows(' + SELECT * FROM tail_fact f, tail_dim d + WHERE d.label IN (''common_a'', ''other_a'') AND d.id = f.dim_id +'); + +-- Test 2: single non-MCV value (same query as baseline). +SELECT * FROM check_estimated_rows(' + SELECT * FROM tail_fact f, tail_dim d + WHERE d.label = ''other_a'' AND d.id = f.dim_id +'); + +DROP STATISTICS jstats_tail; +DROP TABLE tail_fact, tail_dim; + +-- +-- Non-MCV value estimation -- multi-key case. +-- Estimates should not be worse than without join stats. +-- + +CREATE TABLE tail_dim2 ( + id INTEGER PRIMARY KEY, + color TEXT NOT NULL +) WITH (autovacuum_enabled = off); + +CREATE TABLE tail_fact2 ( + dim_id INTEGER NOT NULL, + category TEXT NOT NULL +) WITH (autovacuum_enabled = off); + +INSERT INTO tail_dim2 VALUES + (1, 'common'), (2, 'other'), + (3, 'extra_1'), (4, 'extra_2'); -- dimension-only + +-- 4 combinations = 2 categories x 2 colors; top 2 are both on 'common'. +INSERT INTO tail_fact2 (dim_id, category) +SELECT 1, 'A' FROM generate_series(1, 60) -- (common, A) +UNION ALL SELECT 1, 'B' FROM generate_series(1, 55) -- (common, B) +UNION ALL SELECT 2, 'A' FROM generate_series(1, 43) -- (other, A) +UNION ALL SELECT 2, 'B' FROM generate_series(1, 42); -- (other, B) + +ANALYZE tail_dim2; +ANALYZE tail_fact2; + +-- Baselines without join stat. +SELECT * FROM check_estimated_rows(' + SELECT * FROM tail_fact2 f, tail_dim2 d + WHERE d.color IN (''common'', ''other'') AND d.id = f.dim_id +'); + +SELECT * FROM check_estimated_rows(' + SELECT * FROM tail_fact2 f, tail_dim2 d + WHERE d.color = ''other'' AND d.id = f.dim_id +'); + +-- stattarget=2: only (A,common) and (B,common) land in the MCV. +CREATE STATISTICS jstats_tail2 (mcv) ON f.category, d.color +FROM tail_fact2 f JOIN tail_dim2 d ON (f.dim_id = d.id); +ALTER STATISTICS jstats_tail2 SET STATISTICS 2; +ANALYZE tail_fact2; + +-- Test 1: IN-list mixing MCV + non-MCV color (same query as baseline). +SELECT * FROM check_estimated_rows(' + SELECT * FROM tail_fact2 f, tail_dim2 d + WHERE d.color IN (''common'', ''other'') AND d.id = f.dim_id +'); + +-- Test 2: single non-MCV value (same query as baseline). +SELECT * FROM check_estimated_rows(' + SELECT * FROM tail_fact2 f, tail_dim2 d + WHERE d.color = ''other'' AND d.id = f.dim_id +'); + +DROP STATISTICS jstats_tail2; +DROP TABLE tail_fact2, tail_dim2; + +-- +-- Cross-type join: float4 anchor probing a float8 index. +-- The join uses the float48eq cross-type operator in btree/float_ops. +-- Data is skewed: 80% of fact rows reference 'red' dimension rows. +-- +CREATE TABLE ct_dim (id float8 PRIMARY KEY, color text NOT NULL) + WITH (autovacuum_enabled = off); +CREATE TABLE ct_fact (id serial, dim_id float4 NOT NULL) + WITH (autovacuum_enabled = off); + +-- 15 dimension rows: ids 1-5 are 'red', 6-10 'blue', 11-15 'green'. +INSERT INTO ct_dim SELECT i::float8, CASE WHEN i <= 5 THEN 'red' + WHEN i <= 10 THEN 'blue' + ELSE 'green' END +FROM generate_series(1, 15) i; + +-- 500 fact rows: 80% reference red (ids 1-5), 20% reference others. +-- Standard estimator assumes uniform and underestimates the red share. +INSERT INTO ct_fact (dim_id) +SELECT CASE WHEN i % 5 < 4 THEN (i % 5 + 1)::float4 + ELSE (6 + i % 10)::float4 END +FROM generate_series(1, 500) i; + +ANALYZE ct_dim; +ANALYZE ct_fact; + +-- Baseline: without join stat, standard estimator assumes uniform. +SELECT * FROM check_estimated_rows(' + SELECT * FROM ct_fact f, ct_dim d + WHERE f.dim_id = d.id AND d.color = ''red'' +'); + +CREATE STATISTICS jstats_crosstype (mcv) ON ct_fact.dim_id, ct_dim.color +FROM ct_fact JOIN ct_dim ON (ct_fact.dim_id = ct_dim.id); + +ANALYZE ct_fact; + +-- The stat should be populated (stxdmcv IS NOT NULL). +SELECT stxname FROM pg_statistic_ext +WHERE stxname = 'jstats_crosstype' +AND EXISTS ( + SELECT 1 FROM pg_statistic_ext_data d + WHERE d.stxoid = pg_statistic_ext.oid + AND d.stxdmcv IS NOT NULL +); + +-- With join stat, estimate should improve for the skewed data. +SELECT * FROM check_estimated_rows(' + SELECT * FROM ct_fact f, ct_dim d + WHERE f.dim_id = d.id AND d.color = ''red'' +'); + +DROP STATISTICS jstats_crosstype; +DROP TABLE ct_fact, ct_dim; + +-- +-- Same-named tables in different schemas: deparse must disambiguate aliases. +-- + +CREATE SCHEMA m5_s1; +CREATE SCHEMA m5_s2; + +CREATE TABLE m5_s1.t (a int PRIMARY KEY) WITH (autovacuum_enabled = off); +CREATE TABLE m5_s2.t (b int) WITH (autovacuum_enabled = off); +CREATE INDEX ON m5_s2.t (b); + +INSERT INTO m5_s1.t SELECT generate_series(1, 100); +INSERT INTO m5_s2.t SELECT generate_series(1, 100); + +ANALYZE m5_s1.t; +ANALYZE m5_s2.t; + +CREATE STATISTICS m5_jstats (mcv) ON m5_s2.t.b +FROM m5_s1.t JOIN m5_s2.t ON (m5_s1.t.a = m5_s2.t.b); + +-- The deparsed DDL must use distinct aliases for identically-named tables. +SELECT pg_get_statisticsobjdef(oid) FROM pg_statistic_ext +WHERE stxname = 'm5_jstats'; + +DROP STATISTICS m5_jstats; +DROP TABLE m5_s1.t, m5_s2.t; +DROP SCHEMA m5_s1; +DROP SCHEMA m5_s2; -- 2.50.1 (Apple Git-155)