From 2ab61f9ced64cdc43cb659a6230bf42e7b81783e Mon Sep 17 00:00:00 2001 From: Mikhail Nikalayeu Date: Sun, 11 Jan 2026 19:27:52 +0300 Subject: [PATCH v40 2/7] Add STIR access method and validate_index flag in IndexVacuumInfo This patch provides infrastructure for following enhancements to concurrent index builds by: - validate_index in IndexVacuumInfo: set if index_bulk_delete called during the validation phase of concurrent index build - STIR (Short-Term Index Replacement) access method is introduced, intended solely for short-lived, auxiliary usage STIR functions are designed as an ephemeral helper during concurrent index builds, temporarily storing TIDs without providing the full features of a typical access method. As such, it raises warnings or errors when accessed outside its specialized usage path. Notable implementation points: - STIR indexes are always created unlogged, but the init-fork metapage is WAL-logged: it is the template the main fork is reset to after a crash, and it disables inserts into such leftover indexes - inserts are spread across several insertion pages (with the current page cached in rd_amcache) to avoid lock contention on a single insertion point Planned to be used in following commits. --- contrib/pgstattuple/pgstattuple.c | 3 + src/backend/access/Makefile | 1 + src/backend/access/heap/vacuumlazy.c | 2 + src/backend/access/meson.build | 1 + src/backend/access/stir/Makefile | 18 + src/backend/access/stir/meson.build | 5 + src/backend/access/stir/stir.c | 620 +++++++++++++++++++++++ src/backend/catalog/index.c | 1 + src/backend/commands/analyze.c | 1 + src/backend/commands/vacuumparallel.c | 1 + src/include/access/genam.h | 1 + src/include/access/stir.h | 118 +++++ src/include/catalog/pg_am.dat | 3 + src/include/catalog/pg_opclass.dat | 4 + src/include/catalog/pg_opfamily.dat | 2 + src/include/catalog/pg_proc.dat | 4 + src/include/utils/index_selfuncs.h | 8 + src/test/regress/expected/amutils.out | 8 +- src/test/regress/expected/opr_sanity.out | 9 +- src/test/regress/expected/psql.out | 24 +- src/test/regress/sql/opr_sanity.sql | 2 + src/tools/pgindent/typedefs.list | 4 + 22 files changed, 826 insertions(+), 14 deletions(-) create mode 100644 src/backend/access/stir/Makefile create mode 100644 src/backend/access/stir/meson.build create mode 100644 src/backend/access/stir/stir.c create mode 100644 src/include/access/stir.h diff --git a/contrib/pgstattuple/pgstattuple.c b/contrib/pgstattuple/pgstattuple.c index 6a7f8cb4a7..5b5984e3aa 100644 --- a/contrib/pgstattuple/pgstattuple.c +++ b/contrib/pgstattuple/pgstattuple.c @@ -285,6 +285,9 @@ pgstat_relation(Relation rel, FunctionCallInfo fcinfo) case SPGIST_AM_OID: err = "spgist index"; break; + case STIR_AM_OID: + err = "stir index"; + break; case BRIN_AM_OID: err = "brin index"; break; diff --git a/src/backend/access/Makefile b/src/backend/access/Makefile index e88d72ea03..ebbcfa9071 100644 --- a/src/backend/access/Makefile +++ b/src/backend/access/Makefile @@ -19,6 +19,7 @@ SUBDIRS = \ nbtree \ rmgrdesc \ spgist \ + stir \ sequence \ table \ tablesample \ diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 3e2fc8f0b2..7af12a92dc 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -3054,6 +3054,7 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat, ivinfo.message_level = DEBUG2; ivinfo.num_heap_tuples = reltuples; ivinfo.strategy = vacrel->bstrategy; + ivinfo.validate_index = false; /* Report which index we're currently processing */ pgstat_progress_update_param(PROGRESS_VACUUM_CURRENT_INDEX_RELID, @@ -3118,6 +3119,7 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat, ivinfo.num_heap_tuples = reltuples; ivinfo.strategy = vacrel->bstrategy; + ivinfo.validate_index = false; /* Report which index we're currently processing */ pgstat_progress_update_param(PROGRESS_VACUUM_CURRENT_INDEX_RELID, diff --git a/src/backend/access/meson.build b/src/backend/access/meson.build index 5fd18de74f..7219c65f36 100644 --- a/src/backend/access/meson.build +++ b/src/backend/access/meson.build @@ -11,6 +11,7 @@ subdir('nbtree') subdir('rmgrdesc') subdir('sequence') subdir('spgist') +subdir('stir') subdir('table') subdir('tablesample') subdir('transam') diff --git a/src/backend/access/stir/Makefile b/src/backend/access/stir/Makefile new file mode 100644 index 0000000000..8785dab37b --- /dev/null +++ b/src/backend/access/stir/Makefile @@ -0,0 +1,18 @@ +#------------------------------------------------------------------------- +# +# Makefile-- +# Makefile for access/stir +# +# IDENTIFICATION +# src/backend/access/stir/Makefile +# +#------------------------------------------------------------------------- + +subdir = src/backend/access/stir +top_builddir = ../../../.. +include $(top_builddir)/src/Makefile.global + +OBJS = \ + stir.o + +include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/access/stir/meson.build b/src/backend/access/stir/meson.build new file mode 100644 index 0000000000..4b7ad15346 --- /dev/null +++ b/src/backend/access/stir/meson.build @@ -0,0 +1,5 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +backend_sources += files( + 'stir.c', +) diff --git a/src/backend/access/stir/stir.c b/src/backend/access/stir/stir.c new file mode 100644 index 0000000000..ea14bab66c --- /dev/null +++ b/src/backend/access/stir/stir.c @@ -0,0 +1,620 @@ +/*------------------------------------------------------------------------- + * + * stir.c + * Implementation of Short-Term Index Replacement. + * + * STIR is a specialized access method type designed for temporary storage + * of TID values during concurrent index build operations. + * + * The typical lifecycle of a STIR index is: + * 1. created as an auxiliary index for CIC/RIC + * 2. accepts inserts for a period + * 3. stirbulkdelete called during index validation phase + * 4. gets dropped + * + * STIR indexes are always created as unlogged relations: an auxiliary index + * has no reason to survive a crash, because a crash aborts the concurrent + * index build it was serving, leaving the index as an invalid leftover that + * is only fit to be dropped. Nothing in this AM writes WAL (the asserts on + * !RelationNeedsWAL() in page-modifying paths verify that invariant), with + * one exception: the init-fork metapage written by stirbuildempty(). The + * init fork is the template the main fork is reset to after a crash, so it + * must be durable; its metapage disables inserts, so a reset leftover index + * silently ignores new tuples until it gets dropped. + * + * Portions Copyright (c) 2026, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/backend/access/stir/stir.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/htup_details.h" +#include "access/stir.h" +#include "access/tableam.h" +#include "catalog/index.h" +#include "catalog/pg_amop.h" +#include "catalog/pg_opclass.h" +#include "catalog/pg_opfamily.h" +#include "commands/vacuum.h" +#include "miscadmin.h" +#include "storage/bufmgr.h" +#include "storage/bulk_write.h" +#include "storage/procnumber.h" +#include "utils/catcache.h" +#include "utils/fmgrprotos.h" +#include "utils/index_selfuncs.h" +#include "utils/memutils.h" +#include "utils/regproc.h" +#include "utils/syscache.h" + +/* + * Stir handler function: return IndexAmRoutine with access method parameters + * and callbacks. + */ +Datum +stirhandler(PG_FUNCTION_ARGS) +{ + IndexAmRoutine *amroutine = makeNode(IndexAmRoutine); + + /* Set STIR-specific strategy and procedure numbers */ + amroutine->amstrategies = STIR_NSTRATEGIES; + amroutine->amsupport = STIR_NPROC; + amroutine->amoptsprocnum = STIR_OPTIONS_PROC; + + /* STIR doesn't support most index operations */ + amroutine->amcanorder = false; + amroutine->amcanorderbyop = false; + amroutine->amcanbackward = false; + amroutine->amcanunique = false; + amroutine->amcanmulticol = true; + amroutine->amoptionalkey = true; + amroutine->amsearcharray = false; + amroutine->amsearchnulls = false; + amroutine->amstorage = false; + amroutine->amclusterable = false; + amroutine->ampredlocks = false; + amroutine->amcanparallel = false; + amroutine->amcanbuildparallel = false; + amroutine->amcaninclude = true; + amroutine->amusemaintenanceworkmem = false; + amroutine->amparallelvacuumoptions = VACUUM_OPTION_NO_PARALLEL; + amroutine->amkeytype = InvalidOid; + + /* Set up function callbacks */ + amroutine->ambuild = stirbuild; + amroutine->ambuildempty = stirbuildempty; + amroutine->aminsert = stirinsert; + amroutine->aminsertcleanup = NULL; + amroutine->ambulkdelete = stirbulkdelete; + amroutine->amvacuumcleanup = stirvacuumcleanup; + amroutine->amcanreturn = NULL; + amroutine->amcostestimate = stircostestimate; + amroutine->amoptions = stiroptions; + amroutine->amproperty = NULL; + amroutine->ambuildphasename = NULL; + amroutine->amvalidate = stirvalidate; + amroutine->amadjustmembers = NULL; + amroutine->ambeginscan = stirbeginscan; + amroutine->amrescan = stirrescan; + amroutine->amgettuple = NULL; + amroutine->amgetbitmap = NULL; + amroutine->amendscan = stirendscan; + amroutine->ammarkpos = NULL; + amroutine->amrestrpos = NULL; + amroutine->amestimateparallelscan = NULL; + amroutine->aminitparallelscan = NULL; + amroutine->amparallelrescan = NULL; + + PG_RETURN_POINTER(amroutine); +} + +/* + * Validates operator class for STIR index. + * + * STIR is not a real index, so validate may be skipped. + * But we do it just for consistency. + */ +bool +stirvalidate(Oid opclassoid) +{ + bool result = true; + HeapTuple classtup; + Form_pg_opclass classform; + Oid opfamilyoid; + HeapTuple familytup; + Form_pg_opfamily familyform; + char *opfamilyname; + CatCList *oprlist; + int i; + + /* Fetch opclass information */ + classtup = SearchSysCache1(CLAOID, ObjectIdGetDatum(opclassoid)); + if (!HeapTupleIsValid(classtup)) + elog(ERROR, "cache lookup failed for operator class %u", opclassoid); + classform = (Form_pg_opclass) GETSTRUCT(classtup); + + opfamilyoid = classform->opcfamily; + + /* Fetch opfamily information */ + familytup = SearchSysCache1(OPFAMILYOID, ObjectIdGetDatum(opfamilyoid)); + if (!HeapTupleIsValid(familytup)) + elog(ERROR, "cache lookup failed for operator family %u", opfamilyoid); + familyform = (Form_pg_opfamily) GETSTRUCT(familytup); + + opfamilyname = NameStr(familyform->opfname); + + /* Fetch all operators and support functions of the opfamily */ + oprlist = SearchSysCacheList1(AMOPSTRATEGY, ObjectIdGetDatum(opfamilyoid)); + + /* Check individual operators */ + for (i = 0; i < oprlist->n_members; i++) + { + HeapTuple oprtup = &oprlist->members[i]->tuple; + Form_pg_amop oprform = (Form_pg_amop) GETSTRUCT(oprtup); + + /* STIR carries no key data, so any operator is out of place */ + ereport(INFO, + (errcode(ERRCODE_INVALID_OBJECT_DEFINITION), + errmsg("stir opfamily %s should not contain any operators, but found %s", + opfamilyname, + format_operator(oprform->amopopr)))); + result = false; + } + + ReleaseCatCacheList(oprlist); + ReleaseSysCache(familytup); + ReleaseSysCache(classtup); + + return result; +} + +/* + * Initialize meta-page of a STIR index. + * The skipInserts flag determines if new inserts will be accepted or skipped. + */ +void +StirFillMetapage(Relation index, Page metaPage, bool skipInserts) +{ + StirMetaPageData *metadata; + + StirInitPage(metaPage, STIR_META); + metadata = StirPageGetMeta(metaPage); + memset(metadata, 0, sizeof(StirMetaPageData)); + metadata->magicNumber = STIR_MAGIC_NUMBER; + metadata->skipInserts = skipInserts; + ((PageHeader) metaPage)->pd_lower = ((char *) metadata + sizeof(StirMetaPageData)) - (char *) metaPage; +} + +/* + * Create and initialize the metapage of a STIR index in the main fork. + * This is called during index creation. + */ +void +StirInitMetapage(Relation index) +{ + Buffer metaBuffer; + Page metaPage; + + Assert(!RelationNeedsWAL(index)); + + /* + * Make a new page; since it is the first page it should be associated + * with block number 0 (STIR_METAPAGE_BLKNO). No need to hold the + * extension lock because there cannot be concurrent inserters yet. + */ + metaBuffer = ReadBufferExtended(index, MAIN_FORKNUM, P_NEW, RBM_NORMAL, NULL); + LockBuffer(metaBuffer, BUFFER_LOCK_EXCLUSIVE); + Assert(BufferGetBlockNumber(metaBuffer) == STIR_METAPAGE_BLKNO); + + metaPage = BufferGetPage(metaBuffer); + StirFillMetapage(index, metaPage, false); + + MarkBufferDirty(metaBuffer); + UnlockReleaseBuffer(metaBuffer); +} + +/* + * Initialize any page of a stir index. + */ +void +StirInitPage(Page page, uint16 flags) +{ + StirPageOpaque opaque; + + PageInit(page, BLCKSZ, sizeof(StirPageOpaqueData)); + + opaque = StirPageGetOpaque(page); + opaque->flags = flags; + opaque->stir_page_id = STIR_PAGE_ID; +} + +/* + * Add a tuple to a STIR page. Returns false if the tuple doesn't fit. + * The tuple is added to the end of the page. + */ +static bool +StirPageAddItem(Page page, StirTuple *tuple) +{ + StirTuple *itup; + StirPageOpaque opaque; + char *ptr; + + /* We shouldn't be pointed to an invalid page */ + Assert(!PageIsNew(page)); + + /* Does the new tuple fit on the page? */ + if (StirPageGetFreeSpace(page) < sizeof(StirTuple)) + return false; + + /* Copy a new tuple to the end of the page */ + opaque = StirPageGetOpaque(page); + itup = StirPageGetTuple(page, opaque->maxoff + 1); + memcpy(itup, tuple, sizeof(StirTuple)); + + /* Adjust maxoff and pd_lower */ + opaque->maxoff++; + ptr = (char *) StirPageGetTuple(page, opaque->maxoff + 1); + ((PageHeader) page)->pd_lower = ptr - page; + + /* Assert we didn't overrun available space */ + Assert(((PageHeader) page)->pd_lower <= ((PageHeader) page)->pd_upper); + return true; +} + +/* + * Insert a new tuple into a STIR index. + * + * To avoid contention on a single insertion point, the metapage maintains + * STIR_INSERT_SLOTS independent insertion pages; each backend picks a slot + * by its ProcNumber. In addition, the current insertion page of the slot + * is cached in rd_amcache, so the common path does not touch the metapage + * at all. + * + * A backend with a warm cache may keep inserting into its page for a while + * after skipInserts has been set: that is fine, the flag is only a safety + * net for leftover indexes. The regular way to stop inserts during + * concurrent index builds is clearing indisready, and that invalidates the + * relcache entry, which also resets rd_amcache. + */ +bool +stirinsert(Relation index, Datum *values, bool *isnull, + ItemPointer ht_ctid, Relation heapRel, + IndexUniqueCheck checkUnique, + bool indexUnchanged, + struct IndexInfo *indexInfo) +{ + StirTuple itup; + StirMetaPageData *metaData; + Buffer buffer, + metaBuffer; + Page page; + BlockNumber blkNo; + BlockNumber triedBlkNo = 0; + BlockNumber *cachedBlkNo; + int myslot; + + itup.heapPtr = *ht_ctid; + + Assert(!RelationNeedsWAL(index)); + Assert(MyProcNumber >= 0); + myslot = MyProcNumber % STIR_INSERT_SLOTS; + + /* Lazily set up the per-backend cache of our slot's insertion page. */ + if (index->rd_amcache == NULL) + index->rd_amcache = MemoryContextAllocZero(index->rd_indexcxt, + sizeof(BlockNumber)); + cachedBlkNo = (BlockNumber *) index->rd_amcache; + + /* Fast path: try the cached insertion page, bypassing the metapage. */ + if (*cachedBlkNo > 0) + { + buffer = ReadBuffer(index, *cachedBlkNo); + LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); + + page = BufferGetPage(buffer); + + Assert(!PageIsNew(page)); + + if (StirPageAddItem(page, &itup)) + { + MarkBufferDirty(buffer); + UnlockReleaseBuffer(buffer); + return false; + } + + UnlockReleaseBuffer(buffer); + + /* The page is full; remember it to not try it again below. */ + triedBlkNo = *cachedBlkNo; + *cachedBlkNo = 0; + } + + metaBuffer = ReadBuffer(index, STIR_METAPAGE_BLKNO); + + for (;;) + { + LockBuffer(metaBuffer, BUFFER_LOCK_SHARE); + metaData = StirPageGetMeta(BufferGetPage(metaBuffer)); + /* Check if inserts are allowed */ + if (metaData->skipInserts) + { + UnlockReleaseBuffer(metaBuffer); + return false; + } + blkNo = metaData->slotBlkNo[myslot]; + /* Don't hold metabuffer lock while doing insert */ + LockBuffer(metaBuffer, BUFFER_LOCK_UNLOCK); + + if (blkNo > 0 && blkNo != triedBlkNo) + { + buffer = ReadBuffer(index, blkNo); + LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); + + page = BufferGetPage(buffer); + + Assert(!PageIsNew(page)); + + /* Try to add tuple to the existing page */ + if (StirPageAddItem(page, &itup)) + { + /* Success! Apply the change, clean up, and exit */ + MarkBufferDirty(buffer); + + UnlockReleaseBuffer(buffer); + ReleaseBuffer(metaBuffer); + + *cachedBlkNo = blkNo; + return false; + } + + UnlockReleaseBuffer(buffer); + + /* The page is full, remember that */ + triedBlkNo = blkNo; + } + + /* Need to add a new page - get exclusive lock on meta-page */ + LockBuffer(metaBuffer, BUFFER_LOCK_EXCLUSIVE); + + metaData = StirPageGetMeta(BufferGetPage(metaBuffer)); + + /* Re-check after acquiring exclusive lock */ + if (metaData->skipInserts) + { + UnlockReleaseBuffer(metaBuffer); + return false; + } + + /* Check if another backend already extended the slot */ + if (blkNo != metaData->slotBlkNo[myslot]) + { + Assert(blkNo < metaData->slotBlkNo[myslot]); + + /* + * Someone else inserted the new page into the slot, let's try + * again + */ + LockBuffer(metaBuffer, BUFFER_LOCK_UNLOCK); + } + else + { + /* Must extend the file */ + buffer = ExtendBufferedRel(BMR_REL(index), MAIN_FORKNUM, NULL, + EB_LOCK_FIRST); + page = BufferGetPage(buffer); + + StirInitPage(page, 0); + + if (!StirPageAddItem(page, &itup)) + { + /* We shouldn't be here since we're inserting to an empty page */ + elog(ERROR, "could not add new stir tuple to empty page"); + } + + /* Update meta-page with the new slot block number */ + metaData->slotBlkNo[myslot] = BufferGetBlockNumber(buffer); + *cachedBlkNo = metaData->slotBlkNo[myslot]; + + MarkBufferDirty(metaBuffer); + MarkBufferDirty(buffer); + + UnlockReleaseBuffer(buffer); + UnlockReleaseBuffer(metaBuffer); + + return false; + } + } +} + +/* + * STIR doesn't support scans - these functions all error out + */ +IndexScanDesc +stirbeginscan(Relation r, int nkeys, int norderbys) +{ + ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("\"%s\" is not implemented", __func__))); +} + +void +stirrescan(IndexScanDesc scan, ScanKey scankey, int nscankeys, + ScanKey orderbys, int norderbys) +{ + ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("\"%s\" is not implemented", __func__))); +} + +void +stirendscan(IndexScanDesc scan) +{ + ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("\"%s\" is not implemented", __func__))); +} + +/* + * Build a STIR index - not supported so far. + * Following commits will allow it for auxiliary indexes only. + */ +IndexBuildResult * +stirbuild(Relation heap, Relation index, + struct IndexInfo *indexInfo) +{ + ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("Building STIR indexes is not supported"))); + + return NULL; /* keep compiler quiet */ +} + +/* + * Build an empty STIR index in the initialization fork. + * + * The init fork must be durable: it is what the main fork is reset to after + * a crash. Write the metapage through the bulk-write facility, which + * WAL-logs and syncs init-fork pages regardless of relation persistence. + * The metapage has skipInserts = true, so the reset leftover index silently + * ignores inserts (it is only fit to be dropped anyway). + */ +void +stirbuildempty(Relation index) +{ + BulkWriteState *bulkstate; + BulkWriteBuffer metabuf; + + bulkstate = smgr_bulk_start_rel(index, INIT_FORKNUM); + + metabuf = smgr_bulk_get_buf(bulkstate); + StirFillMetapage(index, (Page) metabuf, true); + smgr_bulk_write(bulkstate, STIR_METAPAGE_BLKNO, metabuf, true); + + smgr_bulk_finish(bulkstate); +} + +IndexBulkDeleteResult * +stirbulkdelete(IndexVacuumInfo *info, + IndexBulkDeleteResult *stats, + IndexBulkDeleteCallback callback, + void *callback_state) +{ + Relation index = info->index; + BlockNumber blkno, + npages; + Buffer buffer; + Page page; + + /* + * For normal VACUUM, mark to skip inserts and warn about an index drop + * needed. In practice this path is not reachable during CREATE INDEX + * CONCURRENTLY because the table-level locks held by CIC prevent + * concurrent VACUUM from opening the auxiliary index. It can only be + * reached if a leftover STIR index somehow survives after a failed CIC + * and a later VACUUM encounters it. + */ + if (!info->validate_index) + { + StirMarkAsSkipInserts(index); + + ereport(WARNING, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("\"%s\" is not implemented, seems like this index needs to be dropped", __func__))); + return NULL; + } + + if (stats == NULL) + stats = (IndexBulkDeleteResult *) palloc0(sizeof(IndexBulkDeleteResult)); + + /* + * Iterate over the pages. We don't care about concurrently added pages, + * because the index is marked as not-ready for that moment and the index + * is not used for insert. + */ + npages = RelationGetNumberOfBlocks(index); + for (blkno = STIR_HEAD_BLKNO; blkno < npages; blkno++) + { + StirTuple *itup, + *itupEnd; + + vacuum_delay_point(false); + + buffer = ReadBufferExtended(index, MAIN_FORKNUM, blkno, + RBM_NORMAL, info->strategy); + + LockBuffer(buffer, BUFFER_LOCK_SHARE); + page = BufferGetPage(buffer); + + if (PageIsNew(page)) + { + UnlockReleaseBuffer(buffer); + continue; + } + + itup = StirPageGetTuple(page, FirstOffsetNumber); + itupEnd = StirPageGetTuple(page, OffsetNumberNext(StirPageGetMaxOffset(page))); + while (itup < itupEnd) + { + /* Do we have to delete this tuple? */ + if (callback(&itup->heapPtr, callback_state)) + { + ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("we never delete in stir"))); + } + + itup = StirPageGetNextTuple(itup); + } + + UnlockReleaseBuffer(buffer); + } + + return stats; +} + +/* + * Mark a STIR index to skip future inserts + */ +void +StirMarkAsSkipInserts(Relation index) +{ + StirMetaPageData *metaData; + Buffer metaBuffer; + Page metaPage; + + Assert(!RelationNeedsWAL(index)); + metaBuffer = ReadBuffer(index, STIR_METAPAGE_BLKNO); + LockBuffer(metaBuffer, BUFFER_LOCK_EXCLUSIVE); + + metaPage = BufferGetPage(metaBuffer); + metaData = StirPageGetMeta(metaPage); + + if (!metaData->skipInserts) + { + metaData->skipInserts = true; + MarkBufferDirty(metaBuffer); + } + UnlockReleaseBuffer(metaBuffer); +} + +/* + * As with stirbulkdelete, this is not reachable during a normal CIC due to + * table-level locking. It serves as a safety net for leftover STIR indexes + * from failed concurrent index builds. + */ +IndexBulkDeleteResult * +stirvacuumcleanup(IndexVacuumInfo *info, + IndexBulkDeleteResult *stats) +{ + StirMarkAsSkipInserts(info->index); + ereport(WARNING, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("\"%s\" is not implemented, seems like this index needs to be dropped", __func__))); + return NULL; +} + +bytea * +stiroptions(Datum reloptions, bool validate) +{ + return NULL; +} + +void +stircostestimate(PlannerInfo *root, IndexPath *path, + double loop_count, Cost *indexStartupCost, + Cost *indexTotalCost, Selectivity *indexSelectivity, + double *indexCorrelation, double *indexPages) +{ + ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("\"%s\" is not implemented", __func__))); +} diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c index 4d232b85ad..1669826475 100644 --- a/src/backend/catalog/index.c +++ b/src/backend/catalog/index.c @@ -3535,6 +3535,7 @@ validate_index(Oid heapId, Oid indexId, Snapshot snapshot) ivinfo.message_level = DEBUG2; ivinfo.num_heap_tuples = heapRelation->rd_rel->reltuples; ivinfo.strategy = NULL; + ivinfo.validate_index = true; /* * Encode TIDs as int8 values for the sort, rather than directly sorting diff --git a/src/backend/commands/analyze.c b/src/backend/commands/analyze.c index 518d752625..bd9b04db7d 100644 --- a/src/backend/commands/analyze.c +++ b/src/backend/commands/analyze.c @@ -753,6 +753,7 @@ do_analyze_rel(Relation onerel, const VacuumParams *params, ivinfo.message_level = elevel; ivinfo.num_heap_tuples = onerel->rd_rel->reltuples; ivinfo.strategy = vac_strategy; + ivinfo.validate_index = false; stats = index_vacuum_cleanup(&ivinfo, NULL); diff --git a/src/backend/commands/vacuumparallel.c b/src/backend/commands/vacuumparallel.c index c29f62c5a9..239b846242 100644 --- a/src/backend/commands/vacuumparallel.c +++ b/src/backend/commands/vacuumparallel.c @@ -1149,6 +1149,7 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel, ivinfo.estimated_count = pvs->shared->estimated_count; ivinfo.num_heap_tuples = pvs->shared->reltuples; ivinfo.strategy = pvs->bstrategy; + ivinfo.validate_index = false; /* Update error traceback information */ pvs->indname = pstrdup(RelationGetRelationName(indrel)); diff --git a/src/include/access/genam.h b/src/include/access/genam.h index 1bfdc1ef6b..6d1301a9d6 100644 --- a/src/include/access/genam.h +++ b/src/include/access/genam.h @@ -57,6 +57,7 @@ typedef struct IndexVacuumInfo bool is_autovacuum; /* VACUUM/ANALYZE initiated by autovacuum */ bool report_progress; /* emit progress.h status reports */ bool estimated_count; /* num_heap_tuples is an estimate */ + bool validate_index; /* validating concurrently built index? */ int message_level; /* ereport level for progress messages */ double num_heap_tuples; /* tuples remaining in heap */ BufferAccessStrategy strategy; /* access strategy for reads */ diff --git a/src/include/access/stir.h b/src/include/access/stir.h new file mode 100644 index 0000000000..fed2b89372 --- /dev/null +++ b/src/include/access/stir.h @@ -0,0 +1,118 @@ +/*------------------------------------------------------------------------- + * + * stir.h + * header file for postgres stir access method implementation. + * + * + * Portions Copyright (c) 2026, PostgreSQL Global Development Group + * + * src/include/access/stir.h + * + *------------------------------------------------------------------------- + */ +#ifndef STIR_H +#define STIR_H + +#include "access/amapi.h" +#include "nodes/pathnodes.h" +#include "storage/bufpage.h" + +/* Support procedures numbers */ +#define STIR_NPROC 0 + +/* Scan strategies (none: STIR carries no key data, so no operators) */ +#define STIR_NSTRATEGIES 0 + +#define STIR_OPTIONS_PROC 0 + +/* Macros for accessing stir page structures */ +#define StirPageGetOpaque(page) ((StirPageOpaque) PageGetSpecialPointer(page)) +#define StirPageGetMaxOffset(page) (StirPageGetOpaque(page)->maxoff) +#define StirPageIsMeta(page) \ + ((StirPageGetOpaque(page)->flags & STIR_META) != 0) +#define StirPageGetTuple(page, offset) \ + ((StirTuple *)(PageGetContents(page) \ + + sizeof(StirTuple) * ((offset) - 1))) +#define StirPageGetNextTuple(tuple) \ + ((StirTuple *)((char *)(tuple) + sizeof(StirTuple))) + + + +/* Preserved page numbers */ +#define STIR_METAPAGE_BLKNO (0) +#define STIR_HEAD_BLKNO (1) /* first data page */ + + +/* Opaque for stir pages */ +typedef struct StirPageOpaqueData +{ + OffsetNumber maxoff; /* number of index tuples on the page */ + uint16 flags; /* see bit definitions below */ + uint16 stir_page_id; /* for identification of STIR indexes */ +} StirPageOpaqueData; + +/* Stir page flags */ +#define STIR_META (1<<0) + +typedef StirPageOpaqueData *StirPageOpaque; + +#define STIR_PAGE_ID 0xFF84 + +/* + * Number of independent insertion pages the inserts are spread across, + * to avoid contention on a single insertion point. Each backend picks + * a slot by its ProcNumber. + */ +#define STIR_INSERT_SLOTS 16 + +/* Metadata of stir index */ +typedef struct StirMetaPageData +{ + uint32 magicNumber; + /* current insertion page of each slot, 0 while the slot has no page */ + BlockNumber slotBlkNo[STIR_INSERT_SLOTS]; + bool skipInserts; /* should we just exit without any inserts? */ +} StirMetaPageData; + +/* Magic number to distinguish stir pages from others */ +#define STIR_MAGIC_NUMBER (0xDBAC0DEF) + +#define StirPageGetMeta(page) ((StirMetaPageData *) PageGetContents(page)) + +typedef struct StirTuple +{ + ItemPointerData heapPtr; +} StirTuple; + +#define StirPageGetFreeSpace(page) \ + (BLCKSZ - MAXALIGN(SizeOfPageHeaderData) \ + - StirPageGetMaxOffset(page) * (sizeof(StirTuple)) \ + - MAXALIGN(sizeof(StirPageOpaqueData))) + +extern void StirFillMetapage(Relation index, Page metaPage, bool skipInserts); +extern void StirInitMetapage(Relation index); +extern void StirInitPage(Page page, uint16 flags); +extern void StirMarkAsSkipInserts(Relation index); + +/* index access method interface functions */ +extern bool stirvalidate(Oid opclassoid); +extern bool stirinsert(Relation index, Datum *values, bool *isnull, + ItemPointer ht_ctid, Relation heapRel, + IndexUniqueCheck checkUnique, + bool indexUnchanged, + struct IndexInfo *indexInfo); +extern IndexScanDesc stirbeginscan(Relation r, int nkeys, int norderbys); +extern void stirrescan(IndexScanDesc scan, ScanKey scankey, int nscankeys, + ScanKey orderbys, int norderbys); +extern void stirendscan(IndexScanDesc scan); +extern IndexBuildResult *stirbuild(Relation heap, Relation index, + struct IndexInfo *indexInfo); +extern void stirbuildempty(Relation index); +extern IndexBulkDeleteResult *stirbulkdelete(IndexVacuumInfo *info, + IndexBulkDeleteResult *stats, IndexBulkDeleteCallback callback, + void *callback_state); +extern IndexBulkDeleteResult *stirvacuumcleanup(IndexVacuumInfo *info, + IndexBulkDeleteResult *stats); +extern bytea *stiroptions(Datum reloptions, bool validate); + +#endif /* STIR_H */ diff --git a/src/include/catalog/pg_am.dat b/src/include/catalog/pg_am.dat index 46d361047f..8bd2c2b46b 100644 --- a/src/include/catalog/pg_am.dat +++ b/src/include/catalog/pg_am.dat @@ -33,5 +33,8 @@ { oid => '3580', oid_symbol => 'BRIN_AM_OID', descr => 'block range index (BRIN) access method', amname => 'brin', amhandler => 'brinhandler', amtype => 'i' }, +{ oid => '5555', oid_symbol => 'STIR_AM_OID', + descr => 'short term index replacement access method', + amname => 'stir', amhandler => 'stirhandler', amtype => 'i' }, ] diff --git a/src/include/catalog/pg_opclass.dat b/src/include/catalog/pg_opclass.dat index 76bb444022..ee013fe34c 100644 --- a/src/include/catalog/pg_opclass.dat +++ b/src/include/catalog/pg_opclass.dat @@ -493,4 +493,8 @@ # no brin opclass for the geometric types except box +# allow any types for STIR +{ opcmethod => 'stir', oid_symbol => 'ANY_STIR_OPS_OID', opcname => 'stir_ops', + opcfamily => 'stir/any_ops', opcintype => 'any'}, + ] diff --git a/src/include/catalog/pg_opfamily.dat b/src/include/catalog/pg_opfamily.dat index c611018310..44a10b1f08 100644 --- a/src/include/catalog/pg_opfamily.dat +++ b/src/include/catalog/pg_opfamily.dat @@ -308,5 +308,7 @@ opfmethod => 'hash', opfname => 'multirange_ops' }, { oid => '6158', opfmethod => 'gist', opfname => 'multirange_ops' }, +{ oid => '5558', + opfmethod => 'stir', opfname => 'any_ops' }, ] diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 1f21a8eb6d..a14e5727ae 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -935,6 +935,10 @@ proname => 'brinhandler', provolatile => 'v', prorettype => 'index_am_handler', proargtypes => 'internal', prosrc => 'brinhandler' }, +{ oid => '5556', descr => 'short term index replacement access method handler', + proname => 'stirhandler', provolatile => 'v', + prorettype => 'index_am_handler', proargtypes => 'internal', + prosrc => 'stirhandler' }, { oid => '3952', descr => 'brin: standalone scan new table pages', proname => 'brin_summarize_new_values', provolatile => 'v', proparallel => 'u', prorettype => 'int4', proargtypes => 'regclass', diff --git a/src/include/utils/index_selfuncs.h b/src/include/utils/index_selfuncs.h index 74793a1a19..bf0e30dabe 100644 --- a/src/include/utils/index_selfuncs.h +++ b/src/include/utils/index_selfuncs.h @@ -62,6 +62,14 @@ extern void spgcostestimate(struct PlannerInfo *root, Selectivity *indexSelectivity, double *indexCorrelation, double *indexPages); +extern void stircostestimate(struct PlannerInfo *root, + struct IndexPath *path, + double loop_count, + Cost *indexStartupCost, + Cost *indexTotalCost, + Selectivity *indexSelectivity, + double *indexCorrelation, + double *indexPages); extern void gincostestimate(struct PlannerInfo *root, struct IndexPath *path, double loop_count, diff --git a/src/test/regress/expected/amutils.out b/src/test/regress/expected/amutils.out index 7ab6113c61..92c033a201 100644 --- a/src/test/regress/expected/amutils.out +++ b/src/test/regress/expected/amutils.out @@ -173,7 +173,13 @@ select amname, prop, pg_indexam_has_property(a.oid, prop) as p spgist | can_exclude | t spgist | can_include | t spgist | bogus | -(36 rows) + stir | can_order | f + stir | can_unique | f + stir | can_multi_col | t + stir | can_exclude | f + stir | can_include | t + stir | bogus | +(42 rows) -- -- additional checks for pg_index_column_has_property diff --git a/src/test/regress/expected/opr_sanity.out b/src/test/regress/expected/opr_sanity.out index 907d63daa3..0d955691a1 100644 --- a/src/test/regress/expected/opr_sanity.out +++ b/src/test/regress/expected/opr_sanity.out @@ -2129,14 +2129,17 @@ WHERE a1.amopopr = o1.oid AND a1.amoppurpose = 's' AND -- Check that each opclass in an opfamily has associated operators, that is -- ones whose oprleft matches opcintype (possibly by coercion). +-- Currently, the only exception is stir_ops: STIR stores no key data and +-- supports no scans, so its opclass intentionally has no operators. SELECT c1.opcname, c1.opcfamily FROM pg_opclass AS c1 WHERE NOT EXISTS(SELECT 1 FROM pg_amop AS a1 WHERE a1.amopfamily = c1.opcfamily AND binary_coercible(c1.opcintype, a1.amoplefttype)); - opcname | opcfamily ----------+----------- -(0 rows) + opcname | opcfamily +----------+----------- + stir_ops | 5558 +(1 row) -- Check that each operator listed in pg_amop has an associated opclass, -- that is one whose opcintype matches oprleft (possibly by coercion). diff --git a/src/test/regress/expected/psql.out b/src/test/regress/expected/psql.out index e8605dd041..1c71a9d377 100644 --- a/src/test/regress/expected/psql.out +++ b/src/test/regress/expected/psql.out @@ -5213,7 +5213,8 @@ List of access methods heap | Table heap2 | Table spgist | Index -(8 rows) + stir | Index +(9 rows) \dA * List of access methods @@ -5227,7 +5228,8 @@ List of access methods heap | Table heap2 | Table spgist | Index -(8 rows) + stir | Index +(9 rows) \dA h* List of access methods @@ -5252,9 +5254,9 @@ List of access methods \dA: extra argument "bar" ignored \dA+ - List of access methods - Name | Type | Handler | Description ---------+-------+----------------------+---------------------------------------- + List of access methods + Name | Type | Handler | Description +--------+-------+----------------------+-------------------------------------------- brin | Index | brinhandler | block range index (BRIN) access method btree | Index | bthandler | b-tree index access method gin | Index | ginhandler | GIN index access method @@ -5263,12 +5265,13 @@ List of access methods heap | Table | heap_tableam_handler | heap table access method heap2 | Table | heap_tableam_handler | spgist | Index | spghandler | SP-GiST index access method -(8 rows) + stir | Index | stirhandler | short term index replacement access method +(9 rows) \dA+ * - List of access methods - Name | Type | Handler | Description ---------+-------+----------------------+---------------------------------------- + List of access methods + Name | Type | Handler | Description +--------+-------+----------------------+-------------------------------------------- brin | Index | brinhandler | block range index (BRIN) access method btree | Index | bthandler | b-tree index access method gin | Index | ginhandler | GIN index access method @@ -5277,7 +5280,8 @@ List of access methods heap | Table | heap_tableam_handler | heap table access method heap2 | Table | heap_tableam_handler | spgist | Index | spghandler | SP-GiST index access method -(8 rows) + stir | Index | stirhandler | short term index replacement access method +(9 rows) \dA+ h* List of access methods diff --git a/src/test/regress/sql/opr_sanity.sql b/src/test/regress/sql/opr_sanity.sql index 5670bb1419..6df1e204e6 100644 --- a/src/test/regress/sql/opr_sanity.sql +++ b/src/test/regress/sql/opr_sanity.sql @@ -1276,6 +1276,8 @@ WHERE a1.amopopr = o1.oid AND a1.amoppurpose = 's' AND -- Check that each opclass in an opfamily has associated operators, that is -- ones whose oprleft matches opcintype (possibly by coercion). +-- Currently, the only exception is stir_ops: STIR stores no key data and +-- supports no scans, so its opclass intentionally has no operators. SELECT c1.opcname, c1.opcfamily FROM pg_opclass AS c1 diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index a7a8dbdd23..ceba0db4de 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -3000,6 +3000,10 @@ StdRdOptIndexCleanup StdRdOptToastValueType StdRdOptions Step +StirMetaPageData +StirPageOpaque +StirPageOpaqueData +StirTuple StopList StrategyNumber StreamCtl -- 2.43.0