From 7a47e2da4712e3f4e6e6fc5f061e6f5d1e55dec1 Mon Sep 17 00:00:00 2001 From: Vaibhav Dalvi Date: Tue, 1 Sep 2026 14:40:12 +0000 Subject: [PATCH] Let gist_trgm_ops estimate cost from real index data Right now gistcostestimate() just guesses the cost for '=' and LIKE queries on gist_trgm_ops, and that guess can be badly wrong. This can make the planner pick GiST over btree for a plain '=' query even when GiST is much slower for it. This patch adds a new optional GiST support function (GIST_COSTESTIMATE_PROC) so an opclass can give the planner a better number. pg_trgm uses it by reading a few pages of its own index, checking how full the signatures really are, and combining that with the number of trigrams in the query to guess how much of the index a scan will actually touch. Opclasses that don't use this function are not affected at all. Since this function is an ordinary SQL-callable one (needed so it can be registered per opclass), it takes its own lock on the index rather than assume one is already held, and checks the index is actually a GiST index over text before reading its pages as trigram signatures. Tested: for a short query with few trigrams, planner now correctly picks btree over GiST. LIKE queries still correctly use GiST when it is genuinely cheaper. Existing pg_trgm tests pass with no changes, and a new test file covers the new function directly. Documentation is updated to note the two user-visible limits below. Known limits: - Only handles simple column = constant conditions for now (no arrays, joins, or other cases where the value isn't a plain constant). - An index without any internal pages yet (which can still cover a fair number of rows) falls back to assuming a full scan, so small tables don't really benefit until the index has grown enough to have real tree levels. - Samples a bounded number of pages on every planning call, with no caching; this adds some repeated I/O to planning, more so for statements that get replanned often. - Treats every internal page the same regardless of its depth in the tree, so the measured density is not a properly traversal-weighted number. - Reads pages with only a shared buffer lock, with no check for a concurrent page split in progress, so a sample can occasionally be skewed under heavy concurrent writes (this can't affect query correctness, only the cost estimate). Discussion: https://postgr.es/m/CAOBaU_YWwtT7tdggtROacjdOdeYHCz-tmSwuC-j-TOG-g97J0w@mail.gmail.com --- contrib/pg_trgm/Makefile | 8 +- .../pg_trgm/expected/pg_trgm_costestimate.out | 67 ++++++++ contrib/pg_trgm/pg_trgm--1.6--1.7.sql | 12 ++ contrib/pg_trgm/pg_trgm.control | 2 +- contrib/pg_trgm/sql/pg_trgm_costestimate.sql | 44 +++++ contrib/pg_trgm/trgm_gist.c | 153 ++++++++++++++++++ doc/src/sgml/pgtrgm.sgml | 7 +- src/backend/access/gist/gistvalidate.c | 7 +- src/backend/utils/adt/selfuncs.c | 92 ++++++++++- src/include/access/gist.h | 11 +- 10 files changed, 391 insertions(+), 12 deletions(-) create mode 100644 contrib/pg_trgm/expected/pg_trgm_costestimate.out create mode 100644 contrib/pg_trgm/pg_trgm--1.6--1.7.sql create mode 100644 contrib/pg_trgm/sql/pg_trgm_costestimate.sql diff --git a/contrib/pg_trgm/Makefile b/contrib/pg_trgm/Makefile index c1756993ec7..767fea6ef82 100644 --- a/contrib/pg_trgm/Makefile +++ b/contrib/pg_trgm/Makefile @@ -9,12 +9,12 @@ OBJS = \ trgm_regexp.o EXTENSION = pg_trgm -DATA = pg_trgm--1.5--1.6.sql pg_trgm--1.4--1.5.sql pg_trgm--1.3--1.4.sql \ - pg_trgm--1.3.sql pg_trgm--1.2--1.3.sql pg_trgm--1.1--1.2.sql \ - pg_trgm--1.0--1.1.sql +DATA = pg_trgm--1.6--1.7.sql pg_trgm--1.5--1.6.sql pg_trgm--1.4--1.5.sql \ + pg_trgm--1.3--1.4.sql pg_trgm--1.3.sql pg_trgm--1.2--1.3.sql \ + pg_trgm--1.1--1.2.sql pg_trgm--1.0--1.1.sql PGFILEDESC = "pg_trgm - trigram matching" -REGRESS = pg_trgm pg_utf8_trgm pg_word_trgm pg_strict_word_trgm +REGRESS = pg_trgm pg_utf8_trgm pg_word_trgm pg_strict_word_trgm pg_trgm_costestimate ifdef USE_PGXS PG_CONFIG = pg_config diff --git a/contrib/pg_trgm/expected/pg_trgm_costestimate.out b/contrib/pg_trgm/expected/pg_trgm_costestimate.out new file mode 100644 index 00000000000..55c71b1e4e2 --- /dev/null +++ b/contrib/pg_trgm/expected/pg_trgm_costestimate.out @@ -0,0 +1,67 @@ +-- Tests for gtrgm_cost_estimate() / GIST_COSTESTIMATE_PROC. +set enable_indexonlyscan = off; +set enable_seqscan = off; +set enable_bitmapscan = off; +create table cost_test (a text); +insert into cost_test select md5(g::text) from generate_series(1, 10000) g; +create index cost_test_btree on cost_test (a); +create index cost_test_gist on cost_test using gist (a gist_trgm_ops); +vacuum freeze analyze cost_test; +-- Low-selectivity, few-trigram value: should not favor the GiST index over +-- btree, since GiST's page-fetch estimate can no longer drop below what +-- gtrgm_cost_estimate() reports for it. +explain (costs off) + select * from cost_test where a = '1234'; + QUERY PLAN +----------------------------------------------- + Index Scan using cost_test_btree on cost_test + Index Cond: (a = '1234'::text) +(2 rows) + +-- Direct calls: bounds, and the strategies we don't model, should be sane +-- and not error. +select gtrgm_cost_estimate('cost_test_gist'::regclass, '1234', 11::smallint) + between 0 and 1 as bounded_equal; + bounded_equal +--------------- + t +(1 row) + +select gtrgm_cost_estimate('cost_test_gist'::regclass, '%ab%', 3::smallint) + between 0 and 1 as bounded_like; + bounded_like +-------------- + t +(1 row) + +select gtrgm_cost_estimate('cost_test_gist'::regclass, 'ab', 1::smallint) + as unmodeled_strategy; + unmodeled_strategy +-------------------- + 0 +(1 row) + +-- A value with more trigrams should never be estimated as touching more of +-- the index than one with fewer trigrams. +select gtrgm_cost_estimate('cost_test_gist'::regclass, '1234', 11::smallint) >= + gtrgm_cost_estimate('cost_test_gist'::regclass, + '123456789012345678901234567890ab', 11::smallint) + as more_trigrams_not_worse; + more_trigrams_not_worse +------------------------- + t +(1 row) + +-- A non-GiST, or GiST-but-not-text, index handed to the function directly +-- must not be misread; it should just report "no data", not error or crash. +select gtrgm_cost_estimate('cost_test_btree'::regclass, 'ab', 11::smallint) + as wrong_index_type; + wrong_index_type +------------------ + 1 +(1 row) + +drop table cost_test; +reset enable_indexonlyscan; +reset enable_seqscan; +reset enable_bitmapscan; diff --git a/contrib/pg_trgm/pg_trgm--1.6--1.7.sql b/contrib/pg_trgm/pg_trgm--1.6--1.7.sql new file mode 100644 index 00000000000..98fa80a30da --- /dev/null +++ b/contrib/pg_trgm/pg_trgm--1.6--1.7.sql @@ -0,0 +1,12 @@ +/* contrib/pg_trgm/pg_trgm--1.6--1.7.sql */ + +-- complain if script is sourced in psql, rather than via ALTER EXTENSION +\echo Use "ALTER EXTENSION pg_trgm UPDATE TO '1.7'" to load this file. \quit + +CREATE FUNCTION gtrgm_cost_estimate(oid, text, smallint) +RETURNS float8 +AS 'MODULE_PATHNAME' +LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; + +ALTER OPERATOR FAMILY gist_trgm_ops USING gist +ADD FUNCTION 13 (text) gtrgm_cost_estimate (oid, text, smallint); diff --git a/contrib/pg_trgm/pg_trgm.control b/contrib/pg_trgm/pg_trgm.control index 1d6a9ddf259..6e3ee43c510 100644 --- a/contrib/pg_trgm/pg_trgm.control +++ b/contrib/pg_trgm/pg_trgm.control @@ -1,6 +1,6 @@ # pg_trgm extension comment = 'text similarity measurement and index searching based on trigrams' -default_version = '1.6' +default_version = '1.7' module_pathname = '$libdir/pg_trgm' relocatable = true trusted = true diff --git a/contrib/pg_trgm/sql/pg_trgm_costestimate.sql b/contrib/pg_trgm/sql/pg_trgm_costestimate.sql new file mode 100644 index 00000000000..a4d87f44818 --- /dev/null +++ b/contrib/pg_trgm/sql/pg_trgm_costestimate.sql @@ -0,0 +1,44 @@ +-- Tests for gtrgm_cost_estimate() / GIST_COSTESTIMATE_PROC. + +set enable_indexonlyscan = off; +set enable_seqscan = off; +set enable_bitmapscan = off; + +create table cost_test (a text); +insert into cost_test select md5(g::text) from generate_series(1, 10000) g; +create index cost_test_btree on cost_test (a); +create index cost_test_gist on cost_test using gist (a gist_trgm_ops); +vacuum freeze analyze cost_test; + +-- Low-selectivity, few-trigram value: should not favor the GiST index over +-- btree, since GiST's page-fetch estimate can no longer drop below what +-- gtrgm_cost_estimate() reports for it. +explain (costs off) + select * from cost_test where a = '1234'; + +-- Direct calls: bounds, and the strategies we don't model, should be sane +-- and not error. +select gtrgm_cost_estimate('cost_test_gist'::regclass, '1234', 11::smallint) + between 0 and 1 as bounded_equal; +select gtrgm_cost_estimate('cost_test_gist'::regclass, '%ab%', 3::smallint) + between 0 and 1 as bounded_like; +select gtrgm_cost_estimate('cost_test_gist'::regclass, 'ab', 1::smallint) + as unmodeled_strategy; + +-- A value with more trigrams should never be estimated as touching more of +-- the index than one with fewer trigrams. +select gtrgm_cost_estimate('cost_test_gist'::regclass, '1234', 11::smallint) >= + gtrgm_cost_estimate('cost_test_gist'::regclass, + '123456789012345678901234567890ab', 11::smallint) + as more_trigrams_not_worse; + +-- A non-GiST, or GiST-but-not-text, index handed to the function directly +-- must not be misread; it should just report "no data", not error or crash. +select gtrgm_cost_estimate('cost_test_btree'::regclass, 'ab', 11::smallint) + as wrong_index_type; + +drop table cost_test; + +reset enable_indexonlyscan; +reset enable_seqscan; +reset enable_bitmapscan; diff --git a/contrib/pg_trgm/trgm_gist.c b/contrib/pg_trgm/trgm_gist.c index 42d0b7a5d65..8208cc6ce8b 100644 --- a/contrib/pg_trgm/trgm_gist.c +++ b/contrib/pg_trgm/trgm_gist.c @@ -3,11 +3,20 @@ */ #include "postgres.h" +#include + +#include "access/genam.h" +#include "access/gist.h" +#include "access/itup.h" #include "access/reloptions.h" #include "access/stratnum.h" +#include "catalog/pg_am_d.h" +#include "catalog/pg_type_d.h" #include "fmgr.h" #include "port/pg_bitutils.h" +#include "storage/bufmgr.h" #include "trgm.h" +#include "utils/rel.h" #include "varatt.h" /* gist_trgm_ops opclass options */ @@ -51,6 +60,7 @@ PG_FUNCTION_INFO_V1(gtrgm_same); PG_FUNCTION_INFO_V1(gtrgm_penalty); PG_FUNCTION_INFO_V1(gtrgm_picksplit); PG_FUNCTION_INFO_V1(gtrgm_options); +PG_FUNCTION_INFO_V1(gtrgm_cost_estimate); Datum @@ -973,3 +983,146 @@ gtrgm_options(PG_FUNCTION_ARGS) PG_RETURN_VOID(); } + +/* Number of internal pages to sample for gtrgm_signature_density(). */ +#define GTRGM_COST_SAMPLE_PAGES 30 + +/* + * Sample internal-page signatures of the given index and return the + * average fraction of their bits that are set, or -1 if nothing could be + * sampled. + */ +static double +gtrgm_signature_density(Oid indexoid, int siglen) +{ + Relation indexRel; + BlockNumber nblocks; + BlockNumber step; + BlockNumber blkno; + int sampledPages = 0; + uint64 setBits = 0; + uint64 totalBits = 0; + double result; + + indexRel = index_open(indexoid, AccessShareLock); + + /* Only a GiST index over text is safe to read as a trigram signature. */ + if (indexRel->rd_rel->relam != GIST_AM_OID || + indexRel->rd_att->natts < 1 || + indexRel->rd_opcintype[0] != TEXTOID) + { + index_close(indexRel, AccessShareLock); + return -1; + } + + nblocks = RelationGetNumberOfBlocks(indexRel); + + if (nblocks <= 1) + { + index_close(indexRel, AccessShareLock); + return -1; + } + + step = Max(1, nblocks / GTRGM_COST_SAMPLE_PAGES); + + for (blkno = 0; blkno < nblocks && sampledPages < GTRGM_COST_SAMPLE_PAGES; + blkno += step) + { + Buffer buf; + Page page; + OffsetNumber maxoff; + OffsetNumber off; + + buf = ReadBuffer(indexRel, blkno); + LockBuffer(buf, BUFFER_LOCK_SHARE); + page = BufferGetPage(buf); + + if (PageIsNew(page) || GistPageIsDeleted(page) || GistPageIsLeaf(page)) + { + UnlockReleaseBuffer(buf); + continue; + } + + sampledPages++; + maxoff = PageGetMaxOffsetNumber(page); + + for (off = FirstOffsetNumber; off <= maxoff; off++) + { + ItemId iid = PageGetItemId(page, off); + IndexTuple itup; + bool isnull; + Datum keyDatum; + TRGM *key; + + if (!ItemIdIsUsed(iid)) + continue; + + itup = (IndexTuple) PageGetItem(page, iid); + keyDatum = index_getattr(itup, 1, RelationGetDescr(indexRel), &isnull); + if (isnull) + continue; + + key = (TRGM *) DatumGetPointer(keyDatum); + totalBits += siglen * BITBYTE; + if (ISALLTRUE(key)) + setBits += siglen * BITBYTE; + else + setBits += pg_popcount((const char *) GETSIGN(key), siglen); + } + + UnlockReleaseBuffer(buf); + } + + index_close(indexRel, AccessShareLock); + + if (totalBits == 0) + return -1; + + result = (double) setBits / (double) totalBits; + return Min(1.0, Max(0.0, result)); +} + +/* + * GIST_COSTESTIMATE_PROC support function. Estimates the fraction of the + * index a scan for the given query and strategy will visit, using the + * query's trigram count together with the index's measured signature + * density as a false-positive-rate model. + */ +Datum +gtrgm_cost_estimate(PG_FUNCTION_ARGS) +{ + Oid indexoid = PG_GETARG_OID(0); + text *query = PG_GETARG_TEXT_P(1); + StrategyNumber strategy = (StrategyNumber) PG_GETARG_UINT16(2); + int siglen = GET_SIGLEN(); + TRGM *trg; + int32 ntrgm; + double density; + + switch (strategy) + { + case EqualStrategyNumber: + trg = generate_trgm(VARDATA(query), VARSIZE(query) - VARHDRSZ); + break; + case ILikeStrategyNumber: + case LikeStrategyNumber: + trg = generate_wildcard_trgm(VARDATA(query), VARSIZE(query) - VARHDRSZ); + break; + default: + /* Haven't modeled this strategy; express no opinion. */ + PG_RETURN_FLOAT8(0.0); + } + + ntrgm = trg ? ARRNELEM(trg) : 0; + if (trg) + pfree(trg); + + if (ntrgm <= 0) + PG_RETURN_FLOAT8(1.0); + + density = gtrgm_signature_density(indexoid, siglen); + if (density < 0) + density = 1.0; /* nothing to sample; assume the worst */ + + PG_RETURN_FLOAT8(pow(density, (double) ntrgm)); +} diff --git a/doc/src/sgml/pgtrgm.sgml b/doc/src/sgml/pgtrgm.sgml index 07bfcac9319..8a796bf5252 100644 --- a/doc/src/sgml/pgtrgm.sgml +++ b/doc/src/sgml/pgtrgm.sgml @@ -421,7 +421,12 @@ pg_trgm. Inequality operators are not supported. Note that those indexes may not be as efficient as regular B-tree indexes - for equality operator. + for equality operator. For gist_trgm_ops, the + planner's cost estimate for an equality search is based on sampling a + few of the index's own pages during planning, so it only reflects the + index's real behavior once the index is large enough to have more than + one tree level, and only for conditions where the compared value is a + simple constant rather than, for example, a join condition. diff --git a/src/backend/access/gist/gistvalidate.c b/src/backend/access/gist/gistvalidate.c index 56feb8d8400..d9bb66cce0d 100644 --- a/src/backend/access/gist/gistvalidate.c +++ b/src/backend/access/gist/gistvalidate.c @@ -144,6 +144,10 @@ gistvalidate(Oid opclassoid) procform->amproclefttype == ANYOID && procform->amprocrighttype == ANYOID; break; + case GIST_COSTESTIMATE_PROC: + ok = check_amproc_signature(procform->amproc, FLOAT8OID, false, + 3, 3, OIDOID, opcintype, INT2OID); + break; default: ereport(INFO, (errcode(ERRCODE_INVALID_OBJECT_DEFINITION), @@ -265,7 +269,7 @@ gistvalidate(Oid opclassoid) if (i == GIST_DISTANCE_PROC || i == GIST_FETCH_PROC || i == GIST_COMPRESS_PROC || i == GIST_DECOMPRESS_PROC || i == GIST_OPTIONS_PROC || i == GIST_SORTSUPPORT_PROC || - i == GIST_TRANSLATE_CMPTYPE_PROC) + i == GIST_TRANSLATE_CMPTYPE_PROC || i == GIST_COSTESTIMATE_PROC) continue; /* optional methods */ ereport(INFO, (errcode(ERRCODE_INVALID_OBJECT_DEFINITION), @@ -337,6 +341,7 @@ gistadjustmembers(Oid opfamilyoid, case GIST_OPTIONS_PROC: case GIST_SORTSUPPORT_PROC: case GIST_TRANSLATE_CMPTYPE_PROC: + case GIST_COSTESTIMATE_PROC: /* Optional, so force it to be a soft family dependency */ op->ref_is_hard = false; op->ref_is_family = true; diff --git a/src/backend/utils/adt/selfuncs.c b/src/backend/utils/adt/selfuncs.c index e27ec9e5c25..dcec0fff302 100644 --- a/src/backend/utils/adt/selfuncs.c +++ b/src/backend/utils/adt/selfuncs.c @@ -100,6 +100,7 @@ #include "access/brin.h" #include "access/brin_page.h" #include "access/gin.h" +#include "access/gist.h" #include "access/table.h" #include "access/tableam.h" #include "access/visibilitymap.h" @@ -8333,6 +8334,89 @@ gistcostestimate(PlannerInfo *root, IndexPath *path, double loop_count, genericcostestimate(root, path, loop_count, &costs); + /* + * If the opclass provides GIST_COSTESTIMATE_PROC, let it refine our + * page-fetch estimate for each Const-qualified indexqual, and don't let + * the estimate go below what it reports. Opclasses that don't provide + * this function are unaffected, as are hypothetical indexes. + */ + if (!index->hypothetical) + { + ListCell *lc; + + foreach(lc, path->indexclauses) + { + IndexClause *iclause = lfirst_node(IndexClause, lc); + int indexcol = iclause->indexcol; + Oid costProcOid; + ListCell *lc2; + + costProcOid = get_opfamily_proc(index->opfamily[indexcol], + index->opcintype[indexcol], + index->opcintype[indexcol], + GIST_COSTESTIMATE_PROC); + if (!OidIsValid(costProcOid)) + continue; + + foreach(lc2, iclause->indexquals) + { + RestrictInfo *rinfo = lfirst_node(RestrictInfo, lc2); + Expr *clause = rinfo->clause; + OpExpr *opclause; + Node *operand; + int strategy; + Oid collation; + FmgrInfo flinfo; + double fraction; + double betterPages; + + /* We only know how to deal with plain two-arg OpExprs here */ + if (!IsA(clause, OpExpr) || + list_length(((OpExpr *) clause)->args) != 2) + continue; + + opclause = (OpExpr *) clause; + operand = (Node *) lsecond(opclause->args); + + /* Aggressively reduce to a constant, looking through casts */ + operand = estimate_expression_value(root, operand); + if (IsA(operand, RelabelType)) + operand = (Node *) ((RelabelType *) operand)->arg; + if (!IsA(operand, Const) || ((Const *) operand)->constisnull) + continue; /* not a case the opclass hook can help with */ + + strategy = get_op_opfamily_strategy(opclause->opno, + index->opfamily[indexcol]); + if (strategy <= 0) + continue; + + if (OidIsValid(index->indexcollations[indexcol])) + collation = index->indexcollations[indexcol]; + else + collation = DEFAULT_COLLATION_OID; + + fmgr_info(costProcOid, &flinfo); + set_fn_opclass_options(&flinfo, index->opclassoptions[indexcol]); + + fraction = DatumGetFloat8(FunctionCall3Coll(&flinfo, + collation, + ObjectIdGetDatum(index->indexoid), + ((Const *) operand)->constvalue, + Int16GetDatum((int16) strategy))); + fraction = Min(1.0, Max(0.0, fraction)); + + betterPages = ceil(fraction * + (index->pages - costs.numNonLeafPages)); + if (betterPages > costs.numIndexPages) + { + costs.indexTotalCost += (betterPages - costs.numIndexPages) * + costs.spc_random_page_cost; + costs.numIndexPages = betterPages; + } + } + } + } + /* * We model index descent costs similarly to those for btree, but to do * that we first need an idea of the tree height. We somewhat arbitrarily @@ -8351,13 +8435,13 @@ gistcostestimate(PlannerInfo *root, IndexPath *path, double loop_count, } /* - * Add a CPU-cost component to represent the costs of initial descent. We - * just use log(N) here not log2(N) since the branching factor isn't - * necessarily two anyway. As for btree, charge once per SA scan. + * Add a CPU-cost component to represent the costs of initial descent. + * Charge about log2(N) comparisons' worth of cost to descend a tree of + * N leaf tuples. As for btree, charge once per SA scan. */ if (index->tuples > 1) /* avoid computing log(0) */ { - descentCost = ceil(log(index->tuples)) * cpu_operator_cost; + descentCost = ceil(log(index->tuples) / log(2.0)) * cpu_operator_cost; costs.indexStartupCost += descentCost; costs.indexTotalCost += costs.num_sa_scans * descentCost; } diff --git a/src/include/access/gist.h b/src/include/access/gist.h index 69a7945c53d..f93e0649401 100644 --- a/src/include/access/gist.h +++ b/src/include/access/gist.h @@ -41,7 +41,16 @@ #define GIST_OPTIONS_PROC 10 #define GIST_SORTSUPPORT_PROC 11 #define GIST_TRANSLATE_CMPTYPE_PROC 12 -#define GISTNProcs 12 + +/* + * Optional support function letting an opclass refine the planner's + * page-fetch estimate for a given strategy and query value. Takes the + * index OID, the query value, and the strategy number, and returns the + * expected fraction of the index a scan will visit. See + * gistcostestimate() in selfuncs.c. + */ +#define GIST_COSTESTIMATE_PROC 13 +#define GISTNProcs 13 /* * Page opaque data in a GiST index page. -- 2.43.0