From 18db23efbc7aa55d58394a1ff58efc13bc1f2b48 Mon Sep 17 00:00:00 2001 From: "ZizhuanLiu(X-MAN)" <44973863@qq.com> Date: Tue, 22 Sep 2026 22:02:12 +0800 Subject: [PATCH v4] Optimize MCV statistics for sortable types by leveraging sorted-order properties 1. Preserve ascending-ordered MCV values (new statistic kind STATISTIC_KIND_MCV_VALUE_SORTED) for sort-comparable types when populating pg_statistic. In compute_scalar_stats(), keep existing logic and allocate an extra ScalarMCVItem workspace to store sorted MCV entries. 2. Use the pre-sorted MCV list during selectivity estimation: - Check against min/max boundaries; boundary cases complete with only 1-2 comparisons. - Entries inside the MCV range use binary search, reducing cost from average N/2 to log(N). - Entries outside the MCV range skip full MCV iteration, limiting comparisons to at most 2. - This optimization is implemented for **var_eq_const()** (equality comparisons) and **mcv_selectivity()** (inequalities: <, <=, >, >=), fully exploiting sorted MCV properties. - mergejoinscansel() calls **get_variable_range()** for both inner and outer relations to retrieve the minimum and maximum values of values[]: when STATISTIC_KIND_MCV_VALUE_SORTED is present, we can avoid the O(n) comparisons on the MCV value[] performed by get_stats_slot_range(). Further functions that can benefit from sorted MCV will be considered later. 3. Completed work: - Compatibility support for non-sortable types and sorted-state detection. - pg_stats view updates to expose STATISTIC_KIND_MCV_VALUE_SORTED MCV values via most_common_vals and most_common_freqs. - verified that we can avoid storing STATISTIC_KIND_MCV_VALUE_SORTED alongside the legacy STATISTIC_KIND_MCV. When compute_scalar_stats() generates the new value-sorted MCV for sortable types, the existing STATISTIC_KIND_MCV entry can be removed or overwritten. 4. TODO: - Audit functions for performance benefits or regressions introduced by sorted MCV, and apply necessary fixes. --- src/backend/catalog/system_views.sql | 10 + src/backend/commands/analyze.c | 13 +- src/backend/executor/nodeHash.c | 6 +- src/backend/statistics/stat_utils.c | 28 + src/backend/utils/adt/like_support.c | 2 +- src/backend/utils/adt/network_selfuncs.c | 26 +- src/backend/utils/adt/selfuncs.c | 746 ++++++++++++++++++++--- src/backend/utils/cache/lsyscache.c | 22 + src/include/catalog/pg_statistic.h | 6 + src/include/statistics/stat_utils.h | 3 + src/include/utils/lsyscache.h | 2 + src/include/utils/selfuncs.h | 2 +- 12 files changed, 769 insertions(+), 97 deletions(-) diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index ad340887..27c22520 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -204,6 +204,11 @@ CREATE VIEW pg_stats WITH (security_barrier) AS WHEN stakind3 = 1 THEN stavalues3 WHEN stakind4 = 1 THEN stavalues4 WHEN stakind5 = 1 THEN stavalues5 + WHEN stakind1 = 8 THEN stavalues1 + WHEN stakind2 = 8 THEN stavalues2 + WHEN stakind3 = 8 THEN stavalues3 + WHEN stakind4 = 8 THEN stavalues4 + WHEN stakind5 = 8 THEN stavalues5 END AS most_common_vals, CASE WHEN stakind1 = 1 THEN stanumbers1 @@ -211,6 +216,11 @@ CREATE VIEW pg_stats WITH (security_barrier) AS WHEN stakind3 = 1 THEN stanumbers3 WHEN stakind4 = 1 THEN stanumbers4 WHEN stakind5 = 1 THEN stanumbers5 + WHEN stakind1 = 8 THEN stanumbers1 + WHEN stakind2 = 8 THEN stanumbers2 + WHEN stakind3 = 8 THEN stanumbers3 + WHEN stakind4 = 8 THEN stanumbers4 + WHEN stakind5 = 8 THEN stanumbers5 END AS most_common_freqs, CASE WHEN stakind1 = 2 THEN stavalues1 diff --git a/src/backend/commands/analyze.c b/src/backend/commands/analyze.c index c05f9f50..fce50331 100644 --- a/src/backend/commands/analyze.c +++ b/src/backend/commands/analyze.c @@ -2481,6 +2481,8 @@ compute_scalar_stats(VacAttrStatsP stats, int values_cnt = 0; int *tupnoLink; ScalarMCVItem *track; + ScalarMCVItem *track_sorted_values; /* tracks values sorted by + * compare_scalars() */ int track_cnt = 0; int num_mcv = stats->attstattarget; int num_bins = stats->attstattarget; @@ -2489,6 +2491,7 @@ compute_scalar_stats(VacAttrStatsP stats, values = palloc_array(ScalarItem, samplerows); tupnoLink = palloc_array(int, samplerows); track = palloc_array(ScalarMCVItem, num_mcv); + track_sorted_values = palloc_array(ScalarMCVItem, num_mcv); memset(&ssup, 0, sizeof(ssup)); ssup.ssup_cxt = CurrentMemoryContext; @@ -2633,6 +2636,9 @@ compute_scalar_stats(VacAttrStatsP stats, } track[j].count = dups_cnt; track[j].first = i + 1 - dups_cnt; + track_sorted_values[track_cnt - 1].count = dups_cnt; + track_sorted_values[track_cnt - 1].first = i + 1 - dups_cnt; + } } dups_cnt = 0; @@ -2776,14 +2782,15 @@ compute_scalar_stats(VacAttrStatsP stats, mcv_freqs = palloc_array(float4, num_mcv); for (i = 0; i < num_mcv; i++) { - mcv_values[i] = datumCopy(values[track[i].first].value, + /* copy in value order */ + mcv_values[i] = datumCopy(values[track_sorted_values[i].first].value, stats->attrtype->typbyval, stats->attrtype->typlen); - mcv_freqs[i] = (double) track[i].count / (double) samplerows; + mcv_freqs[i] = (double) track_sorted_values[i].count / (double) samplerows; } MemoryContextSwitchTo(old_context); - stats->stakind[slot_idx] = STATISTIC_KIND_MCV; + stats->stakind[slot_idx] = STATISTIC_KIND_MCV_VALUE_SORTED; stats->staop[slot_idx] = mystats->eqopr; stats->stacoll[slot_idx] = stats->attrcollid; stats->stanumbers[slot_idx] = mcv_freqs; diff --git a/src/backend/executor/nodeHash.c b/src/backend/executor/nodeHash.c index 8825bb6f..e80e0001 100644 --- a/src/backend/executor/nodeHash.c +++ b/src/backend/executor/nodeHash.c @@ -2449,9 +2449,9 @@ ExecHashBuildSkewHash(HashState *hashstate, HashJoinTable hashtable, if (!HeapTupleIsValid(statsTuple)) return; - if (get_attstatsslot(&sslot, statsTuple, - STATISTIC_KIND_MCV, InvalidOid, - ATTSTATSSLOT_VALUES | ATTSTATSSLOT_NUMBERS)) + if (get_attstatsslot_mcv(&sslot, statsTuple, + InvalidOid, + ATTSTATSSLOT_VALUES | ATTSTATSSLOT_NUMBERS)) { double frac; int nbuckets; diff --git a/src/backend/statistics/stat_utils.c b/src/backend/statistics/stat_utils.c index f4ff9ab9..bc3d7dba 100644 --- a/src/backend/statistics/stat_utils.c +++ b/src/backend/statistics/stat_utils.c @@ -816,3 +816,31 @@ statatt_check_bounds_histogram(Datum arrayval) return true; } + +/* + * max_mcv_numbers + * For compatibility: for STATISTIC_KIND_MCV_VALUE_SORTED MCV stats, + * return the maximum numbers[] entry; return index [0] for STATISTIC_KIND_MCV. + * + * Return 0.0 for non-MCV statistics. + */ +float4 +max_mcv_numbers(AttStatsSlot *sslot, int statskind) +{ + int i = 0; + + if (statskind != STATISTIC_KIND_MCV || + statskind != STATISTIC_KIND_MCV_VALUE_SORTED) + return 0.0; + + if (statskind == STATISTIC_KIND_MCV_VALUE_SORTED) + { + for (int j = 1; j < sslot->nnumbers; j++) + { + if (sslot->numbers[j] > sslot->numbers[i]) + i = j; + } + } + + return sslot->numbers[i]; +} diff --git a/src/backend/utils/adt/like_support.c b/src/backend/utils/adt/like_support.c index 588425b3..0f6d08fe 100644 --- a/src/backend/utils/adt/like_support.c +++ b/src/backend/utils/adt/like_support.c @@ -730,7 +730,7 @@ patternsel_common(PlannerInfo *root, */ mcv_selec = mcv_selectivity(&vardata, &opproc, collation, constval, true, - &sumcommon); + &sumcommon, opfuncid); /* * Now merge the results from the MCV and histogram calculations, diff --git a/src/backend/utils/adt/network_selfuncs.c b/src/backend/utils/adt/network_selfuncs.c index 2a8d2ded..cf15cff0 100644 --- a/src/backend/utils/adt/network_selfuncs.c +++ b/src/backend/utils/adt/network_selfuncs.c @@ -148,7 +148,7 @@ networksel(PG_FUNCTION_ARGS) fmgr_info(get_opcode(operator), &proc); mcv_selec = mcv_selectivity(&vardata, &proc, InvalidOid, constvalue, varonleft, - &sumcommon); + &sumcommon, operator); /* * If we have a histogram, use it to estimate the proportion of the @@ -305,9 +305,9 @@ networkjoinsel_inner(Oid operator, int opr_codenum, stats = (Form_pg_statistic) GETSTRUCT(vardata1->statsTuple); nullfrac1 = stats->stanullfrac; - mcv1_exists = get_attstatsslot(&mcv1_slot, vardata1->statsTuple, - STATISTIC_KIND_MCV, InvalidOid, - ATTSTATSSLOT_VALUES | ATTSTATSSLOT_NUMBERS); + mcv1_exists = get_attstatsslot_mcv(&mcv1_slot, vardata1->statsTuple, + InvalidOid, + ATTSTATSSLOT_VALUES | ATTSTATSSLOT_NUMBERS); hist1_exists = get_attstatsslot(&hist1_slot, vardata1->statsTuple, STATISTIC_KIND_HISTOGRAM, InvalidOid, ATTSTATSSLOT_VALUES); @@ -327,9 +327,9 @@ networkjoinsel_inner(Oid operator, int opr_codenum, stats = (Form_pg_statistic) GETSTRUCT(vardata2->statsTuple); nullfrac2 = stats->stanullfrac; - mcv2_exists = get_attstatsslot(&mcv2_slot, vardata2->statsTuple, - STATISTIC_KIND_MCV, InvalidOid, - ATTSTATSSLOT_VALUES | ATTSTATSSLOT_NUMBERS); + mcv2_exists = get_attstatsslot_mcv(&mcv2_slot, vardata2->statsTuple, + InvalidOid, + ATTSTATSSLOT_VALUES | ATTSTATSSLOT_NUMBERS); hist2_exists = get_attstatsslot(&hist2_slot, vardata2->statsTuple, STATISTIC_KIND_HISTOGRAM, InvalidOid, ATTSTATSSLOT_VALUES); @@ -432,9 +432,9 @@ networkjoinsel_semi(Oid operator, int opr_codenum, stats = (Form_pg_statistic) GETSTRUCT(vardata1->statsTuple); nullfrac1 = stats->stanullfrac; - mcv1_exists = get_attstatsslot(&mcv1_slot, vardata1->statsTuple, - STATISTIC_KIND_MCV, InvalidOid, - ATTSTATSSLOT_VALUES | ATTSTATSSLOT_NUMBERS); + mcv1_exists = get_attstatsslot_mcv(&mcv1_slot, vardata1->statsTuple, + InvalidOid, + ATTSTATSSLOT_VALUES | ATTSTATSSLOT_NUMBERS); hist1_exists = get_attstatsslot(&hist1_slot, vardata1->statsTuple, STATISTIC_KIND_HISTOGRAM, InvalidOid, ATTSTATSSLOT_VALUES); @@ -454,9 +454,9 @@ networkjoinsel_semi(Oid operator, int opr_codenum, stats = (Form_pg_statistic) GETSTRUCT(vardata2->statsTuple); nullfrac2 = stats->stanullfrac; - mcv2_exists = get_attstatsslot(&mcv2_slot, vardata2->statsTuple, - STATISTIC_KIND_MCV, InvalidOid, - ATTSTATSSLOT_VALUES | ATTSTATSSLOT_NUMBERS); + mcv2_exists = get_attstatsslot_mcv(&mcv2_slot, vardata2->statsTuple, + InvalidOid, + ATTSTATSSLOT_VALUES | ATTSTATSSLOT_NUMBERS); hist2_exists = get_attstatsslot(&hist2_slot, vardata2->statsTuple, STATISTIC_KIND_HISTOGRAM, InvalidOid, ATTSTATSSLOT_VALUES); diff --git a/src/backend/utils/adt/selfuncs.c b/src/backend/utils/adt/selfuncs.c index 5ee37759..6760df8a 100644 --- a/src/backend/utils/adt/selfuncs.c +++ b/src/backend/utils/adt/selfuncs.c @@ -117,10 +117,12 @@ #include "optimizer/paths.h" #include "optimizer/plancat.h" #include "parser/parse_clause.h" +#include "parser/parse_oper.h" #include "parser/parse_relation.h" #include "parser/parsetree.h" #include "rewrite/rewriteManip.h" #include "statistics/statistics.h" +#include "statistics/stat_utils.h" #include "utils/acl.h" #include "utils/array.h" #include "utils/builtins.h" @@ -135,6 +137,7 @@ #include "utils/selfuncs.h" #include "utils/snapmgr.h" #include "utils/spccache.h" +#include "utils/sortsupport.h" #include "utils/syscache.h" #include "utils/timestamp.h" #include "utils/typcache.h" @@ -154,6 +157,40 @@ #define EQJOINSEL_MCV_HASH_THRESHOLD 20 #endif +/* + * Invoke the comparison function against a single MCV entry. + * Returns true if comparison succeeds and result is not null. + */ +#define MCV_ENTRY_MATCH(fcinfo, arg_mcv, sslot, i, fresult) \ +( \ + (fcinfo->args[arg_mcv].value = sslot.values[i]), \ + (fcinfo->isnull = false), \ + (fresult = FunctionCallInvoke(fcinfo)), \ + (!fcinfo->isnull && DatumGetBool(fresult)) \ +) + +/* + * Status codes for IN_MCV_RANGE: + * 0 - Must compare against every MCV entry. + * 1 - Within MCV range; still need to check existence + * via binary search or other means. + * 2 - Outside MCV range; no per-MCV comparison required. + */ +#define IN_MCV_RANGE_UNKNOWN 0 +#define IN_MCV_RANGE_YES 1 +#define IN_MCV_RANGE_NO 2 + +/* Threshold: MCV entries worthy of special comparison (e.g. binary search) */ +#define MCV_SPECIAL_COMPARE_THRESHOLD 3 + +/* + * When using "<" in ApplySortComparator(constval, ?, sslot.values[?], ?, ?), + * indicates constval's relative position compared with sslot.values[?]. + */ +#define ON_LEFT(i) ((i) < 0) +#define ON_EQUAL(i) ((i) == 0) +#define ON_RIGHT(i) ((i) > 0) + /* Entries in the simplehash hash table used by eqjoinsel_find_matches */ typedef struct MCVHashEntry { @@ -411,6 +448,12 @@ var_eq_const(VariableStatData *vardata, Oid oproid, Oid collation, AttStatsSlot sslot; bool match = false; int i; + double sumcommon = 0.0; + int statskind; + + statskind = get_attstatsslot_mcv(&sslot, vardata->statsTuple, + InvalidOid, + ATTSTATSSLOT_VALUES | ATTSTATSSLOT_NUMBERS); /* * Is the constant "=" to any of the column's most common values? @@ -419,12 +462,14 @@ var_eq_const(VariableStatData *vardata, Oid oproid, Oid collation, * don't like this, maybe you shouldn't be using eqsel for your * operator...) */ - if (get_attstatsslot(&sslot, vardata->statsTuple, - STATISTIC_KIND_MCV, InvalidOid, - ATTSTATSSLOT_VALUES | ATTSTATSSLOT_NUMBERS)) + if (statskind) { LOCAL_FCINFO(fcinfo, 2); FmgrInfo eqproc; + bool scan_entire_mcv = false; + int in_mcv_range = IN_MCV_RANGE_UNKNOWN; + SortSupportData ssup = {0}; + int arg_mcv; fmgr_info(opfuncoid, &eqproc); @@ -440,24 +485,198 @@ var_eq_const(VariableStatData *vardata, Oid oproid, Oid collation, fcinfo->args[1].isnull = false; /* be careful to apply operator right way 'round */ if (varonleft) + { fcinfo->args[1].value = constval; + arg_mcv = 0; + } else + { fcinfo->args[0].value = constval; + arg_mcv = 1; + } - for (i = 0; i < sslot.nvalues; i++) + selec = 0.0; + i = 0; + + if (sslot.stacoll != collation && OidIsValid(collation)) { - Datum fresult; + /* + * Scanning the entire MCV array is needed when the collation + * used for the comparison is nondeterministic and differs + * from the statistics collation. In this case, the comparison + * may match multiple MCV values, so we must continue scanning + * after finding a match. + */ + pg_locale_t mylocale = pg_newlocale_from_collation(collation); - if (varonleft) - fcinfo->args[0].value = sslot.values[i]; - else - fcinfo->args[1].value = sslot.values[i]; - fcinfo->isnull = false; - fresult = FunctionCallInvoke(fcinfo); - if (!fcinfo->isnull && DatumGetBool(fresult)) + scan_entire_mcv = !mylocale->deterministic; + } + + if (sslot.stacoll == collation && + statskind == STATISTIC_KIND_MCV_VALUE_SORTED && + sslot.nvalues > MCV_SPECIAL_COMPARE_THRESHOLD && + comparison_ops_are_compatible(sslot.staop, oproid)) + { + /* + * If collations match, datatype is sortable and MCV array + * sorted, optimize comparisons using this property: + * + * - If it is equal to the first or last MCV value, the lookup + * can be completed immediately. + * + * - If it falls within the MCV range, use binary search to + * find a matching value, reducing the average number of + * comparisons from N/2 to at most log(N). + * + * - If it falls outside the MCV range, subsequent processing + * can sum sumcommon directly without comparing against the + * MCV values. In this worst-case scenario where the constant + * does not match any MCV value, this reduces the number of + * comparisons from N to at most 2. + * + */ + Oid ltopr; + Oid eqopr; + + /* Look for default "<" and "=" operators for sslot.valuetype */ + get_sort_group_operators(sslot.valuetype, + false, false, false, + <opr, &eqopr, NULL, + NULL); + + if (OidIsValid(eqopr) && OidIsValid(ltopr)) { - match = true; - break; + /* datatype is sortable */ + int compare; + + scan_entire_mcv = false; /* And no full MCV scan + * required */ + + ssup.ssup_cxt = CurrentMemoryContext; + ssup.ssup_collation = sslot.stacoll; + ssup.ssup_nulls_first = false; + ssup.abbreviate = false; + PrepareSortSupportFromOrderingOp(ltopr, &ssup); + + /* First compare against values[0] */ + compare = ApplySortComparator(constval, false, + sslot.values[0], false, &ssup); + if (ON_EQUAL(compare)) + { + /* + * Constant is "=" to this common value. We know + * selectivity exactly (or as exactly as ANALYZE could + * calculate it, anyway). + */ + match = true; + selec = sslot.numbers[0]; + } + else if (ON_LEFT(compare) || sslot.nvalues == 1) + { + in_mcv_range = IN_MCV_RANGE_NO; + } + else + { + /* Next compare against values[sslot.nvalues - 1] */ + compare = ApplySortComparator(constval, false, + sslot.values[sslot.nvalues - 1], false, &ssup); + if (ON_EQUAL(compare)) + { + /* + * Constant is "=" to this common value. We know + * selectivity exactly (or as exactly as ANALYZE + * could calculate it, anyway). + */ + match = true; + selec = sslot.numbers[sslot.nvalues - 1]; + } + else if (ON_RIGHT(compare) || sslot.nvalues == 2) + in_mcv_range = IN_MCV_RANGE_NO; + else if (ON_LEFT(compare)) + { + int tmp_l_bound; + int tmp_r_bound; + + /* + * For binary search: refers to the first and last + * uncompared elements. + */ + tmp_l_bound = 1; + tmp_r_bound = sslot.nvalues - 2; + + while (tmp_l_bound <= tmp_r_bound) + { + int mid = (tmp_l_bound + tmp_r_bound) / 2; + + compare = ApplySortComparator(constval, false, + sslot.values[mid], false, &ssup); + if (ON_EQUAL(compare)) + { + /* + * Constant is "=" to this common value. + * We know selectivity exactly (or as + * exactly as ANALYZE could calculate it, + * anyway). + */ + match = true; + selec = sslot.numbers[mid]; + + break; + } + else if (ON_RIGHT(compare)) + tmp_l_bound = mid + 1; + else if (ON_LEFT(compare)) + tmp_r_bound = mid - 1; + } + + if (!match) + in_mcv_range = IN_MCV_RANGE_NO; + } + } + } + } + + if (!match) + { + /* + * Compare the constant expression with the MCVs. + * + * If the constant matches the current MCV, stop here when a + * full MCV scan is not required. Otherwise, continue scanning + * the remaining MCVs and accumulate the selectivity of all + * matching MCVs. + * + * While scanning, also accumulate the selectivity of the MCVs + * examined so far. This is used later to estimate the + * selectivity of a non-NULL constant that does not match any + * MCV. + */ + for (i = 0; i < sslot.nvalues; i++) + { + if (in_mcv_range == IN_MCV_RANGE_UNKNOWN || scan_entire_mcv) + { + Datum fresult; + + if MCV_ENTRY_MATCH + (fcinfo, arg_mcv, sslot, i, fresult) + { + /* + * Constant is "=" to this common value. We know + * selectivity exactly (or as exactly as ANALYZE + * could calculate it, anyway). + */ + match = true; + if (!scan_entire_mcv) + { + selec = sslot.numbers[i]; + break; + } + + selec += sslot.numbers[i]; + } + } + + sumcommon += sslot.numbers[i]; } } } @@ -467,26 +686,15 @@ var_eq_const(VariableStatData *vardata, Oid oproid, Oid collation, i = 0; /* keep compiler quiet */ } - if (match) - { - /* - * Constant is "=" to this common value. We know selectivity - * exactly (or as exactly as ANALYZE could calculate it, anyway). - */ - selec = sslot.numbers[i]; - } - else + if (!match) { /* * Comparison is against a constant that is neither NULL nor any * of the common values. Its selectivity cannot be more than * this: */ - double sumcommon = 0.0; double otherdistinct; - for (i = 0; i < sslot.nnumbers; i++) - sumcommon += sslot.numbers[i]; selec = 1.0 - sumcommon - nullfrac; CLAMP_PROBABILITY(selec); @@ -570,6 +778,7 @@ var_eq_non_const(VariableStatData *vardata, Oid oproid, Oid collation, { double ndistinct; AttStatsSlot sslot; + int statskind; /* * Search is for a value that we do not know a priori, but we will @@ -590,12 +799,16 @@ var_eq_non_const(VariableStatData *vardata, Oid oproid, Oid collation, * Cross-check: selectivity should never be estimated as more than the * most common value's. */ - if (get_attstatsslot(&sslot, vardata->statsTuple, - STATISTIC_KIND_MCV, InvalidOid, - ATTSTATSSLOT_NUMBERS)) + statskind = get_attstatsslot_mcv(&sslot, vardata->statsTuple, + InvalidOid, + ATTSTATSSLOT_NUMBERS); + if (statskind) { - if (sslot.nnumbers > 0 && selec > sslot.numbers[0]) - selec = sslot.numbers[0]; + double max_numbers; + + max_numbers = max_mcv_numbers(&sslot, statskind); + if (selec > max_numbers) + selec = max_numbers; free_attstatsslot(&sslot); } } @@ -753,7 +966,7 @@ scalarineqsel(PlannerInfo *root, Oid operator, bool isgt, bool iseq, * by MCV entries. */ mcv_selec = mcv_selectivity(vardata, &opproc, collation, constval, true, - &sumcommon); + &sumcommon, operator); /* * If there is a histogram, determine which bin the constant falls in, and @@ -805,23 +1018,29 @@ scalarineqsel(PlannerInfo *root, Oid operator, bool isgt, bool iseq, double mcv_selectivity(VariableStatData *vardata, FmgrInfo *opproc, Oid collation, Datum constval, bool varonleft, - double *sumcommonp) + double *sumcommonp, Oid operator) { double mcv_selec, sumcommon; AttStatsSlot sslot; int i; + int statskind; mcv_selec = 0.0; sumcommon = 0.0; if (HeapTupleIsValid(vardata->statsTuple) && statistic_proc_security_check(vardata, opproc->fn_oid) && - get_attstatsslot(&sslot, vardata->statsTuple, - STATISTIC_KIND_MCV, InvalidOid, - ATTSTATSSLOT_VALUES | ATTSTATSSLOT_NUMBERS)) + (statskind = get_attstatsslot_mcv(&sslot, vardata->statsTuple, + InvalidOid, + ATTSTATSSLOT_VALUES | ATTSTATSSLOT_NUMBERS))) { LOCAL_FCINFO(fcinfo, 2); + Datum fresult; + int in_mcv_range = IN_MCV_RANGE_UNKNOWN; + int lBound = -1; /* -1 means unknown */ + int rBound = -1; /* -1 means unknown */ + int arg_mcv; /* * We invoke the opproc "by hand" so that we won't fail on NULL @@ -836,24 +1055,379 @@ mcv_selectivity(VariableStatData *vardata, FmgrInfo *opproc, Oid collation, fcinfo->args[1].isnull = false; /* be careful to apply operator right way 'round */ if (varonleft) + { fcinfo->args[1].value = constval; + arg_mcv = 0; + } else + { fcinfo->args[0].value = constval; + arg_mcv = 1; + } + + if (sslot.stacoll == collation && + statskind == STATISTIC_KIND_MCV_VALUE_SORTED && + sslot.nvalues > MCV_SPECIAL_COMPARE_THRESHOLD && + comparison_ops_are_compatible(sslot.staop, operator)) + { + /* + * If collations match, datatype is sortable and MCV array sorted, + * optimize comparisons using this property. + */ + Oid ltopr; + Oid eqopr; + + /* Look for default "<" and "=" operators for sslot.valuetype. */ + get_sort_group_operators(sslot.valuetype, + false, false, false, + <opr, &eqopr, NULL, + NULL); + if (OidIsValid(eqopr) && OidIsValid(ltopr)) + { + RegProcedure oprsel; + SortSupportData ssup = {0}; + int compare; + + ssup.ssup_cxt = CurrentMemoryContext; + ssup.ssup_collation = sslot.stacoll; + ssup.ssup_nulls_first = false; + ssup.abbreviate = false; + PrepareSortSupportFromOrderingOp(ltopr, &ssup); + + oprsel = get_oprrest(operator); + if (oprsel == F_SCALARLTSEL || oprsel == F_SCALARLESEL) + { + /* + * For "<" and "<=", compare starting at Min value: + * values[0] + */ + compare = ApplySortComparator(constval, false, + sslot.values[0], false, &ssup); + if (ON_LEFT(compare)) + { + /* + * constval < Min value: all MCV entries < constval, + * no MCV entries satisfy the predicate. + */ + in_mcv_range = IN_MCV_RANGE_NO; + } + else if (ON_EQUAL(compare)) + { + /* constval == Min value */ + if (oprsel == F_SCALARLTSEL) + { + /* + * When oprsel is "<": but all MCV entries < + * constval, so no MCV entries satisfy the + * predicate. + */ + in_mcv_range = IN_MCV_RANGE_NO; + } + else if (oprsel == F_SCALARLESEL) + { + /* + * When oprsel is "<=": only the Min value satisfy + * the predicate. + */ + in_mcv_range = IN_MCV_RANGE_YES; + lBound = 0; + rBound = lBound; + } + } + else if (ON_RIGHT(compare)) + { + /* + * Min value < constval. Left boundary determined, now + * find right boundary. + */ + in_mcv_range = IN_MCV_RANGE_YES; + lBound = 0; + + /* compare at Max value: values[sslot.nvalues - 1] */ + compare = ApplySortComparator(constval, false, + sslot.values[sslot.nvalues - 1], false, &ssup); + if (ON_RIGHT(compare)) + { + /* + * Max value < constval: all MCV entries "<"/"<=" + * constval, all MCV entries satisfy the + * predicate. + */ + rBound = sslot.nvalues - 1; + } + else if (ON_EQUAL(compare)) + { + /* Max value == constval */ + if (oprsel == F_SCALARLTSEL) + { + /* + * when oprsel is "<": values[sslot.nvalues - + * 2] is the rightmost entry satisfying the + * predicate. + */ + rBound = sslot.nvalues - 2; + } + else if (oprsel == F_SCALARLESEL) + { + /* + * When oprsel is "<=": all MCV entries "<=" + * constval, all MCV entries satisfy the + * predicate. + */ + rBound = sslot.nvalues - 1; + } + } + else if ON_LEFT + (compare) + { + /* + * Max value > constval: binary search. + */ + int tmp_l_bound; + int tmp_r_bound; + + /* + * Initially set right boundary equal to left + * boundary + */ + rBound = lBound; + + /* + * For binary search: refers to the 1st and last + * uncompared elements. + */ + tmp_l_bound = lBound + 1; + tmp_r_bound = sslot.nvalues - 2; + + while (tmp_l_bound <= tmp_r_bound) + { + int mid = (tmp_l_bound + tmp_r_bound) / 2; + + compare = ApplySortComparator(constval, false, + sslot.values[mid], false, &ssup); + if (ON_RIGHT(compare)) + { + /* + * values[mid] < constval: continue + * rightward search. + */ + rBound = mid; + tmp_l_bound = mid + 1; + + } + else if (ON_EQUAL(compare)) + { + /* values[mid] = constval */ + if (oprsel == F_SCALARLTSEL) + { + /* + * when oprsel is "<": values[mid - 1] + * is the max one satisfy the + * predicate. + */ + rBound = mid - 1; + } + else if (oprsel == F_SCALARLESEL) + { + /* + * When oprsel is "<=": values[mid] is + * the max one satisfy the predicate. + */ + rBound = mid; + } + + break; /* stop search */ + } + else if (ON_LEFT(compare)) + { + /* continue leftward search */ + tmp_r_bound = mid - 1; + } + } + } + } + } + else if (oprsel == F_SCALARGTSEL || oprsel == F_SCALARGESEL) + { + /* + * For '>' and '>=', compare starting at Max value: + * [sslot.nvalues - 1] + */ + compare = ApplySortComparator(constval, false, + sslot.values[sslot.nvalues - 1], false, &ssup); + if (ON_RIGHT(compare)) + { + /* + * constval > Max value: all MCV entries < constval, + * no MCV entries satisfy the predicate. + */ + in_mcv_range = IN_MCV_RANGE_NO; + } + else if (ON_EQUAL(compare)) + { + /* constval == Max value */ + if (oprsel == F_SCALARGTSEL) + { + /* + * When oprsel is ">": All MCV entries < constval, + * no MCV entries satisfy the predicate. + */ + in_mcv_range = IN_MCV_RANGE_NO; + } + else if (oprsel == F_SCALARGESEL) + { + /* + * When oprsel is ">=": only the Max value + * satisfies the predicate. + */ + in_mcv_range = IN_MCV_RANGE_YES; + rBound = sslot.nvalues - 1; + lBound = rBound; + } + } + else if (ON_LEFT(compare)) + { + /* + * constval < Max value. Right boundary determined, + * now find left boundary. + */ + in_mcv_range = IN_MCV_RANGE_YES; + rBound = sslot.nvalues - 1; + + /* compare at Min value: values[0] */ + compare = ApplySortComparator(constval, false, + sslot.values[0], false, &ssup); + if (ON_LEFT(compare)) + { + /* + * constval < Min value: All MCV entries ">"/">=" + * constval, all MCV entries satisfy the + * predicate. + */ + lBound = 0; + } + else if (ON_EQUAL(compare)) + { + /* constval == Min value */ + if (oprsel == F_SCALARGTSEL) + { + /* + * When oprsel is ">": values[1] is the + * leftmost entry satisfying the predicate. + */ + lBound = 1; + } + else if (oprsel == F_SCALARGESEL) + { + /* + * when oprsel is ">=": All MCV entries ">=" + * constval, all MCV entries satisfy the + * predicate. + */ + lBound = 0; + } + } + else if (ON_RIGHT(compare)) + { + /* + * Min value > constval: binary search. + */ + int tmp_l_bound; + int tmp_r_bound; + + /* + * Initially set left boundary equal to right + * boundary + */ + lBound = rBound; + + /* + * For binary search: refers to the 1st and last + * uncompared elements. + */ + tmp_l_bound = 1; + tmp_r_bound = rBound - 1; + + while (tmp_l_bound <= tmp_r_bound) + { + int mid = (tmp_l_bound + tmp_r_bound) / 2; + + compare = ApplySortComparator(constval, false, + sslot.values[mid], false, &ssup); + + if (ON_LEFT(compare)) + { + /* + * constval < values[mid]: continue + * leftward search. + */ + lBound = mid; + tmp_r_bound = mid - 1; + } + else if (ON_EQUAL(compare)) + { + /* values[mid] == constval */ + if (oprsel == F_SCALARGTSEL) + { + /* + * When oprsel is ">": values[mid + 1] + * is the min one satisfy the + * predicate. + */ + lBound = mid + 1; + } + else if (oprsel == F_SCALARGESEL) + { + /* + * When oprsel is >=: values[mid] is + * the min one satisfy the predicate. + */ + lBound = mid; + } + + break; /* stop search */ + } + else if (ON_RIGHT(compare)) + { + /* continue rightward search */ + tmp_l_bound = mid + 1; + } + } + } + } + } + } + } for (i = 0; i < sslot.nvalues; i++) { - Datum fresult; + sumcommon += sslot.numbers[i]; /* Accumulate sumcommon first */ - if (varonleft) - fcinfo->args[0].value = sslot.values[i]; - else - fcinfo->args[1].value = sslot.values[i]; - fcinfo->isnull = false; - fresult = FunctionCallInvoke(fcinfo); - if (!fcinfo->isnull && DatumGetBool(fresult)) + /* + * If outside the MCV range, no comparison needed; + * + * If already known to be within MCV bounds, only accumulate + * mcv_selec over entries in range with no extra comparisons; + * + * Otherwise, perform on-the-fly comparisons to identify matches + * and accumulate mcv_selec. + */ + + if (in_mcv_range == IN_MCV_RANGE_NO) + continue; + + else if (in_mcv_range == IN_MCV_RANGE_YES) + { + if (lBound <= i && i <= rBound) + mcv_selec += sslot.numbers[i]; + + continue; + } + + if (MCV_ENTRY_MATCH(fcinfo, arg_mcv, sslot, i, fresult)) mcv_selec += sslot.numbers[i]; - sumcommon += sslot.numbers[i]; } + free_attstatsslot(&sslot); } @@ -1030,7 +1604,7 @@ generic_restriction_selectivity(PlannerInfo *root, Oid oproid, Oid collation, */ mcvsel = mcv_selectivity(&vardata, &opproc, collation, constval, varonleft, - &mcvsum); + &mcvsum, oproid); /* * If the histogram is large enough, see what fraction of it matches @@ -1275,9 +1849,9 @@ ineq_histogram_selectivity(PlannerInfo *root, &isdefault); /* Subtract off the number of known MCVs */ - if (get_attstatsslot(&mcvslot, vardata->statsTuple, - STATISTIC_KIND_MCV, InvalidOid, - ATTSTATSSLOT_NUMBERS)) + if (get_attstatsslot_mcv(&mcvslot, vardata->statsTuple, + InvalidOid, + ATTSTATSSLOT_NUMBERS)) { otherdistinct -= mcvslot.nnumbers; free_attstatsslot(&mcvslot); @@ -1639,9 +2213,9 @@ booltestsel(PlannerInfo *root, BoolTestType booltesttype, Node *arg, stats = (Form_pg_statistic) GETSTRUCT(vardata.statsTuple); freq_null = stats->stanullfrac; - if (get_attstatsslot(&sslot, vardata.statsTuple, - STATISTIC_KIND_MCV, InvalidOid, - ATTSTATSSLOT_VALUES | ATTSTATSSLOT_NUMBERS) + if (get_attstatsslot_mcv(&sslot, vardata.statsTuple, + InvalidOid, + ATTSTATSSLOT_VALUES | ATTSTATSSLOT_NUMBERS) && sslot.nnumbers > 0) { double freq_true; @@ -2436,12 +3010,12 @@ eqjoinsel(PG_FUNCTION_ARGS) */ get_mcv_stats = (HeapTupleIsValid(vardata1.statsTuple) && HeapTupleIsValid(vardata2.statsTuple) && - get_attstatsslot(&sslot1, vardata1.statsTuple, - STATISTIC_KIND_MCV, InvalidOid, - 0) && - get_attstatsslot(&sslot2, vardata2.statsTuple, - STATISTIC_KIND_MCV, InvalidOid, - 0)); + get_attstatsslot_mcv(&sslot1, vardata1.statsTuple, + InvalidOid, + 0) && + get_attstatsslot_mcv(&sslot2, vardata2.statsTuple, + InvalidOid, + 0)); if (HeapTupleIsValid(vardata1.statsTuple)) { @@ -2449,9 +3023,9 @@ eqjoinsel(PG_FUNCTION_ARGS) stats1 = (Form_pg_statistic) GETSTRUCT(vardata1.statsTuple); if (get_mcv_stats && statistic_proc_security_check(&vardata1, opfuncoid)) - have_mcvs1 = get_attstatsslot(&sslot1, vardata1.statsTuple, - STATISTIC_KIND_MCV, InvalidOid, - ATTSTATSSLOT_VALUES | ATTSTATSSLOT_NUMBERS); + have_mcvs1 = get_attstatsslot_mcv(&sslot1, vardata1.statsTuple, + InvalidOid, + ATTSTATSSLOT_VALUES | ATTSTATSSLOT_NUMBERS); } if (HeapTupleIsValid(vardata2.statsTuple)) @@ -2460,9 +3034,9 @@ eqjoinsel(PG_FUNCTION_ARGS) stats2 = (Form_pg_statistic) GETSTRUCT(vardata2.statsTuple); if (get_mcv_stats && statistic_proc_security_check(&vardata2, opfuncoid)) - have_mcvs2 = get_attstatsslot(&sslot2, vardata2.statsTuple, - STATISTIC_KIND_MCV, InvalidOid, - ATTSTATSSLOT_VALUES | ATTSTATSSLOT_NUMBERS); + have_mcvs2 = get_attstatsslot_mcv(&sslot2, vardata2.statsTuple, + InvalidOid, + ATTSTATSSLOT_VALUES | ATTSTATSSLOT_NUMBERS); } /* Prepare info usable by both eqjoinsel_inner and eqjoinsel_semi */ @@ -4429,6 +5003,7 @@ estimate_hash_bucket_stats(PlannerInfo *root, Node *hashkey, double nbuckets, ndistinct; bool isdefault; AttStatsSlot sslot; + int statskind; examine_variable(root, hashkey, 0, &vardata); @@ -4438,15 +5013,16 @@ estimate_hash_bucket_stats(PlannerInfo *root, Node *hashkey, double nbuckets, /* Look up the frequency of the most common value, if available */ if (HeapTupleIsValid(vardata.statsTuple)) { - if (get_attstatsslot(&sslot, vardata.statsTuple, - STATISTIC_KIND_MCV, InvalidOid, - ATTSTATSSLOT_NUMBERS)) + statskind = get_attstatsslot_mcv(&sslot, vardata.statsTuple, + InvalidOid, + ATTSTATSSLOT_NUMBERS); + if (statskind) { /* - * The first MCV stat is for the most common value. + * get the most common value. */ if (sslot.nnumbers > 0) - *mcv_freq = sslot.numbers[0]; + *mcv_freq = max_mcv_numbers(&sslot, statskind); free_attstatsslot(&sslot); } else if (get_attstatsslot(&sslot, vardata.statsTuple, @@ -6854,6 +7430,7 @@ get_variable_range(PlannerInfo *root, VariableStatData *vardata, Oid opfuncoid; FmgrInfo opproc; AttStatsSlot sslot; + int statskind; /* * XXX It's very tempting to try to use the actual column min and max, if @@ -6930,10 +7507,11 @@ get_variable_range(PlannerInfo *root, VariableStatData *vardata, * data. Proceed only if the MCVs represent the whole table (to within * roundoff error). */ - if (get_attstatsslot(&sslot, vardata->statsTuple, - STATISTIC_KIND_MCV, InvalidOid, - have_data ? ATTSTATSSLOT_VALUES : - (ATTSTATSSLOT_VALUES | ATTSTATSSLOT_NUMBERS))) + statskind = get_attstatsslot_mcv(&sslot, vardata->statsTuple, + InvalidOid, + have_data ? ATTSTATSSLOT_VALUES : + (ATTSTATSSLOT_VALUES | ATTSTATSSLOT_NUMBERS)); + if (statskind) { bool use_mcvs = have_data; @@ -6951,9 +7529,25 @@ get_variable_range(PlannerInfo *root, VariableStatData *vardata, } if (use_mcvs) - get_stats_slot_range(&sslot, opfuncoid, &opproc, - collation, typLen, typByVal, - &tmin, &tmax, &have_data); + { + /* + * Like STATISTIC_KIND_HISTOGRAM: if the MCV slot is + * STATISTIC_KIND_MCV_VALUE_SORTED, it has the ordering we want, + * grab the last and first values. + */ + if (sslot.stacoll == collation && sslot.nvalues > 0 && + statskind == STATISTIC_KIND_MCV_VALUE_SORTED) + { + tmin = datumCopy(sslot.values[0], typByVal, typLen); + tmax = datumCopy(sslot.values[sslot.nvalues - 1], typByVal, typLen); + have_data = true; + } + else + get_stats_slot_range(&sslot, opfuncoid, &opproc, + collation, typLen, typByVal, + &tmin, &tmax, &have_data); + } + free_attstatsslot(&sslot); } diff --git a/src/backend/utils/cache/lsyscache.c b/src/backend/utils/cache/lsyscache.c index 65b2d645..f1f51766 100644 --- a/src/backend/utils/cache/lsyscache.c +++ b/src/backend/utils/cache/lsyscache.c @@ -3650,6 +3650,28 @@ get_attstatsslot(AttStatsSlot *sslot, HeapTuple statstuple, return true; } +/* + * get_attstatsslot_mcv + * For compatibility, callers not using compute_scalar_stats() to generate + * STATISTIC_KIND_MCV_VALUE_SORTED may still attempt to retrieve STATISTIC_KIND_MCV. + */ +int +get_attstatsslot_mcv(AttStatsSlot *sslot, HeapTuple statstuple, + Oid reqop, int flags) +{ + if (get_attstatsslot(sslot, statstuple, + STATISTIC_KIND_MCV_VALUE_SORTED, reqop, + flags)) + return STATISTIC_KIND_MCV_VALUE_SORTED; + + if (get_attstatsslot(sslot, statstuple, + STATISTIC_KIND_MCV, reqop, + flags)) + return STATISTIC_KIND_MCV; + + return 0; +} + /* * free_attstatsslot * Free data allocated by get_attstatsslot diff --git a/src/include/catalog/pg_statistic.h b/src/include/catalog/pg_statistic.h index 032bf177..42d1aded 100644 --- a/src/include/catalog/pg_statistic.h +++ b/src/include/catalog/pg_statistic.h @@ -291,6 +291,12 @@ DECLARE_FOREIGN_KEY((starelid, staattnum), pg_attribute, (attrelid, attnum)); */ #define STATISTIC_KIND_BOUNDS_HISTOGRAM 7 +/* + * Like STATISTIC_KIND_MCV, except that stanumbersN/stavaluesN pairs + * are sorted in descending order of the stavaluesN datum value. + */ +#define STATISTIC_KIND_MCV_VALUE_SORTED 8 + #endif /* EXPOSE_TO_CLIENT_CODE */ #endif /* PG_STATISTIC_H */ diff --git a/src/include/statistics/stat_utils.h b/src/include/statistics/stat_utils.h index 15e962db..e1a79202 100644 --- a/src/include/statistics/stat_utils.h +++ b/src/include/statistics/stat_utils.h @@ -15,6 +15,7 @@ #include "access/attnum.h" #include "fmgr.h" +#include "utils/lsyscache.h" /* avoid including primnodes.h here */ typedef struct RangeVar RangeVar; @@ -60,4 +61,6 @@ extern bool statatt_get_elem_type(Oid atttypid, char atttyptype, extern bool statatt_check_bounds_histogram(Datum arrayval); +extern float4 max_mcv_numbers(AttStatsSlot *sslot, int reqkind); + #endif /* STATS_UTILS_H */ diff --git a/src/include/utils/lsyscache.h b/src/include/utils/lsyscache.h index 171fbda9..213f2838 100644 --- a/src/include/utils/lsyscache.h +++ b/src/include/utils/lsyscache.h @@ -199,6 +199,8 @@ extern int32 get_typavgwidth(Oid typid, int32 typmod); extern int32 get_attavgwidth(Oid relid, AttrNumber attnum); extern bool get_attstatsslot(AttStatsSlot *sslot, HeapTuple statstuple, int reqkind, Oid reqop, int flags); +extern int get_attstatsslot_mcv(AttStatsSlot *sslot, HeapTuple statstuple, + Oid reqop, int flags); extern void free_attstatsslot(AttStatsSlot *sslot); extern char *get_namespace_name(Oid nspid); extern char *get_namespace_name_or_temp(Oid nspid); diff --git a/src/include/utils/selfuncs.h b/src/include/utils/selfuncs.h index 8d9fff95..e4887358 100644 --- a/src/include/utils/selfuncs.h +++ b/src/include/utils/selfuncs.h @@ -177,7 +177,7 @@ extern double get_variable_numdistinct(VariableStatData *vardata, extern double mcv_selectivity(VariableStatData *vardata, FmgrInfo *opproc, Oid collation, Datum constval, bool varonleft, - double *sumcommonp); + double *sumcommonp, Oid operator); extern double histogram_selectivity(VariableStatData *vardata, FmgrInfo *opproc, Oid collation, Datum constval, bool varonleft, -- 2.43.0