From a2517ae6e46b07e84e44a35a8defa267a2eb606c Mon Sep 17 00:00:00 2001 From: Mikhail Nikalayeu Date: Mon, 12 Oct 2026 12:01:05 +0200 Subject: [PATCH v8 6/7] Allow snapshot resets in concurrent unique index builds With snapshot resets, equal keys collected by the scan may belong to rows that were never live at the same time. Check their current heap status in _bt_load() before reporting a uniqueness violation. Limit heap fetches in tuplesort and _bt_load() using separate budgets, controlled by index_build_duplicate_check_scale_factor (5% by default). Each budget allows at least 1000 fetches unless the setting is zero. Leave unchecked equal-key entries out of the index so validation can insert them with a full uniqueness check. Also leave entries for validation when a possible conflict involves an unfinished delete or update. Validation runs after waiting for older writers, so a concurrent delete can resolve the apparent duplicate. Concurrent unique builds no longer need a separate spool for dead tuples, since the MVCC scan only supplies tuples visible to its current snapshot. Reviewed-by: Matthias van de Meent --- contrib/test_decoding/expected/repack.out | 7 + contrib/test_decoding/sql/repack.sql | 2 + doc/src/sgml/config.sgml | 26 ++ src/backend/access/heap/README.HOT | 12 +- src/backend/access/nbtree/nbtsort.c | 239 ++++++++--- src/backend/access/nbtree/nbtutils.c | 104 +++++ src/backend/catalog/index.c | 11 +- src/backend/commands/indexcmds.c | 12 +- src/backend/utils/misc/guc_parameters.dat | 10 + src/backend/utils/sort/tuplesort.c | 2 + src/backend/utils/sort/tuplesortvariants.c | 91 ++++- src/include/access/nbtree.h | 8 + src/include/miscadmin.h | 1 + src/include/utils/tuplesort.h | 3 + src/test/modules/injection_points/Makefile | 1 + .../expected/cic_reset_snapshots.out | 198 ++++++++- .../expected/cic_reset_unique.out | 381 ++++++++++++++++++ src/test/modules/injection_points/meson.build | 1 + .../specs/cic_reset_unique.spec | 109 +++++ .../sql/cic_reset_snapshots.sql | 129 ++++++ 20 files changed, 1260 insertions(+), 87 deletions(-) create mode 100644 src/test/modules/injection_points/expected/cic_reset_unique.out create mode 100644 src/test/modules/injection_points/specs/cic_reset_unique.spec diff --git a/contrib/test_decoding/expected/repack.out b/contrib/test_decoding/expected/repack.out index e7060a7e57..059832ed39 100644 --- a/contrib/test_decoding/expected/repack.out +++ b/contrib/test_decoding/expected/repack.out @@ -57,6 +57,13 @@ INSERT INTO repack_conc_invidx VALUES (1, 0), (2, 0); CREATE UNIQUE INDEX CONCURRENTLY repack_conc_invidx_uq ON repack_conc_invidx (j); ERROR: could not create unique index "repack_conc_invidx_uq" DETAIL: Key (j)=(0) is duplicated. +SELECT indisready FROM pg_index +WHERE indexrelid = 'repack_conc_invidx_uq'::regclass; + indisready +------------ + f +(1 row) + CREATE INDEX CONCURRENTLY repack_conc_invalid_expr ON repack_conc_invidx ((1/j)); ERROR: division by zero REPACK repack_conc_invidx; diff --git a/contrib/test_decoding/sql/repack.sql b/contrib/test_decoding/sql/repack.sql index 0107e0bf89..a2e5bb9f46 100644 --- a/contrib/test_decoding/sql/repack.sql +++ b/contrib/test_decoding/sql/repack.sql @@ -37,6 +37,8 @@ DROP TABLE rpk_missing; CREATE TABLE repack_conc_invidx (i int PRIMARY KEY, j int); INSERT INTO repack_conc_invidx VALUES (1, 0), (2, 0); CREATE UNIQUE INDEX CONCURRENTLY repack_conc_invidx_uq ON repack_conc_invidx (j); +SELECT indisready FROM pg_index +WHERE indexrelid = 'repack_conc_invidx_uq'::regclass; CREATE INDEX CONCURRENTLY repack_conc_invalid_expr ON repack_conc_invidx ((1/j)); REPACK repack_conc_invidx; REPACK (CONCURRENTLY) repack_conc_invidx; diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 99f54395ce..3c27e08c58 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -13031,6 +13031,32 @@ dynamic_library_path = '/usr/local/lib/postgresql:$libdir' + + index_build_duplicate_check_scale_factor (floating point) + + index_build_duplicate_check_scale_factor configuration parameter + + + + + Limits heap fetches for early duplicate checks while sorting and + loading a concurrent unique index build. Sorting and loading have + separate budgets. Each sort uses a fraction of the tuples received + so far; loading uses a fraction of the total tuples collected by the + scan, including all parallel workers. The default fraction is + 0.05. For a nonzero fraction, each budget allows at + least 1000 heap fetches, so small builds can detect duplicates early. + + + After the load budget is exhausted, further entries with equal keys + are omitted from the index. Validation inserts any surviving tuples + with a full uniqueness check. A value of 0 leaves + duplicate checks to validation. This limit does not apply to heap + fetches during validation. + + + + post_auth_delay (integer) diff --git a/src/backend/access/heap/README.HOT b/src/backend/access/heap/README.HOT index 74e407f375..e87c521b7c 100644 --- a/src/backend/access/heap/README.HOT +++ b/src/backend/access/heap/README.HOT @@ -386,12 +386,12 @@ have the HOT-safety property enforced before we start to build the new index. After waiting for transactions which had the table open, we build the index -for all rows that are valid in a fresh snapshot. Any tuples visible in the -snapshot will have only valid forward-growing HOT chains. (They might have -older HOT updates behind them which are broken, but this is OK for the same -reason it's OK in a regular index build.) As above, we point the index -entry at the root of the HOT-update chain but we use the key value from the -live tuple. +from rows visible to the scan, which may replace its snapshot periodically. +Any tuples visible to the scan have only valid forward-growing HOT chains. +(They might have older HOT updates behind them which are broken, but this is +OK for the same reason it's OK in a regular index build.) +As above, we point the index entry at the root of the HOT-update chain but we +use the key value from the live tuple. We mark the index open for inserts (but still not ready for reads) then we again wait for transactions which have the table open. Then we take diff --git a/src/backend/access/nbtree/nbtsort.c b/src/backend/access/nbtree/nbtsort.c index 6f9db97d99..093cd4ebff 100644 --- a/src/backend/access/nbtree/nbtsort.c +++ b/src/backend/access/nbtree/nbtsort.c @@ -55,6 +55,7 @@ #include "storage/condition_variable.h" #include "storage/proc.h" #include "tcop/tcopprot.h" +#include "utils/injection_point.h" #include "utils/rel.h" #include "utils/sortsupport.h" #include "utils/tuplesort.h" @@ -69,6 +70,14 @@ #define PARALLEL_KEY_WAL_USAGE UINT64CONST(0xA000000000000005) #define PARALLEL_KEY_BUFFER_USAGE UINT64CONST(0xA000000000000006) +/* + * Only non-concurrent unique builds need a separate spool to exclude dead + * tuples from uniqueness checks. Concurrent scans collect live tuples, and + * their checks account for tuples that die during the build. + */ +#define BTBuildNeedsSpool2(isunique, isconcurrent) \ + ((isunique) && !(isconcurrent)) + /* * DISABLE_LEADER_PARTICIPATION disables the leader's participation in * parallel index builds. This may be useful as a debugging aid. @@ -87,7 +96,8 @@ typedef struct BTSpool Relation index; bool isunique; bool nulls_not_distinct; - bool reset_snapshot; /* from ii_ResetSnapshot; leader only */ + bool unique_dead_ignored; + int64 heap_fetch_limit; /* heap fetches allowed during loading */ } BTSpool; /* @@ -106,6 +116,7 @@ typedef struct BTShared Oid indexrelid; bool isunique; bool nulls_not_distinct; + bool unique_dead_ignored; bool isconcurrent; int scantuplesortstates; @@ -189,13 +200,11 @@ typedef struct BTLeader * btshared is the shared state for entire build. sharedsort is the * shared, tuplesort-managed state passed to each process tuplesort. * sharedsort2 is the corresponding btspool2 shared state, used only when - * building unique indexes. snapshot is the snapshot used by the scan iff - * an MVCC snapshot is required. + * a build spools dead tuples separately (see BTBuildNeedsSpool2()). */ BTShared *btshared; Sharedsort *sharedsort; Sharedsort *sharedsort2; - Snapshot snapshot; WalUsage *walusage; BufferUsage *bufferusage; } BTLeader; @@ -208,15 +217,13 @@ typedef struct BTLeader */ typedef struct BTBuildState { - bool isunique; - bool nulls_not_distinct; bool havedead; Relation heap; BTSpool *spool; /* - * spool2 is needed only when the index is a unique index. Dead tuples are - * put into spool2 instead of spool in order to avoid uniqueness check. + * Non-concurrent unique builds put dead tuples in spool2 to exclude them + * from uniqueness checks. */ BTSpool *spool2; double indtuples; @@ -308,8 +315,6 @@ btbuild(Relation heap, Relation index, IndexInfo *indexInfo) ResetUsage(); #endif /* BTREE_BUILD_STATS */ - buildstate.isunique = indexInfo->ii_Unique; - buildstate.nulls_not_distinct = indexInfo->ii_NullsNotDistinct; buildstate.havedead = false; buildstate.heap = heap; buildstate.spool = NULL; @@ -384,7 +389,14 @@ _bt_spools_heapscan(Relation heap, Relation index, BTBuildState *buildstate, btspool->index = index; btspool->isunique = indexInfo->ii_Unique; btspool->nulls_not_distinct = indexInfo->ii_NullsNotDistinct; - btspool->reset_snapshot = indexInfo->ii_ResetSnapshot; + + /* + * Concurrent scans can replace their snapshots, so uniqueness checks must + * account for tuples that die after the scan collects them. + */ + Assert(indexInfo->ii_ResetSnapshot == indexInfo->ii_Concurrent); + btspool->unique_dead_ignored = indexInfo->ii_Concurrent && + indexInfo->ii_Unique; /* Save as primary spool */ buildstate->spool = btspool; @@ -433,17 +445,18 @@ _bt_spools_heapscan(Relation heap, Relation index, BTBuildState *buildstate, * the use of parallelism or any other factor. */ buildstate->spool->sortstate = - tuplesort_begin_index_btree(heap, index, buildstate->isunique, - buildstate->nulls_not_distinct, + tuplesort_begin_index_btree(heap, index, btspool->isunique, + btspool->nulls_not_distinct, + btspool->unique_dead_ignored, maintenance_work_mem, coordinate, TUPLESORT_NONE); /* - * If building a unique index, put dead tuples in a second spool to keep - * them out of the uniqueness check. We expect that the second spool (for - * dead tuples) won't get very full, so we give it only work_mem. + * Non-concurrent unique builds need a second spool for dead tuples. It + * is usually small, so allow it only work_mem. Concurrent builds use + * MVCC snapshots and collect only tuples visible to the scan. */ - if (indexInfo->ii_Unique) + if (BTBuildNeedsSpool2(indexInfo->ii_Unique, indexInfo->ii_Concurrent)) { BTSpool *btspool2 = palloc0_object(BTSpool); SortCoordinate coordinate2 = NULL; @@ -474,7 +487,7 @@ _bt_spools_heapscan(Relation heap, Relation index, BTBuildState *buildstate, * full, so we give it only work_mem */ buildstate->spool2->sortstate = - tuplesort_begin_index_btree(heap, index, false, false, work_mem, + tuplesort_begin_index_btree(heap, index, false, false, false, work_mem, coordinate2, TUPLESORT_NONE); } @@ -487,6 +500,13 @@ _bt_spools_heapscan(Relation heap, Relation index, BTBuildState *buildstate, reltuples = _bt_parallel_heapscan(buildstate, &indexInfo->ii_BrokenHotChain); + /* The load budget uses the total from all scan participants. */ + btspool->heap_fetch_limit = (int64) + (buildstate->indtuples * index_build_duplicate_check_scale_factor); + /* Give small builds enough budget to catch duplicates early. */ + if (index_build_duplicate_check_scale_factor > 0) + btspool->heap_fetch_limit = Max(btspool->heap_fetch_limit, 1000); + /* * Set the progress target for the next phase. Reset the block number * values set by table_index_build_scan @@ -1157,7 +1177,134 @@ _bt_load(BTWriteState *wstate, BTSpool *btspool, BTSpool *btspool2) deduplicate = wstate->inskey->allequalimage && !btspool->isunique && BTGetDeduplicateItems(wstate->index); - if (merge) + if (btspool->unique_dead_ignored) + { + /* + * Check duplicates within the load budget, then leave unchecked + * entries for validation. Equal keys collected under different + * snapshots need not conflict. + */ + HTSV_Result seen_status = HEAPTUPLE_DEAD; /* group status */ + bool in_group = false; /* is prev in a group of equal keys? */ + IndexTuple prev = NULL; /* copy of the previous tuple */ + Size prevbufsize = 0; /* size of the buffer holding it */ + int64 heap_fetches_left = btspool->heap_fetch_limit; + + Assert(btspool->isunique); + Assert(!btspool2); + + while ((itup = tuplesort_getindextuple(btspool->sortstate, true)) != NULL) + { + bool defer_to_validation = false; + + /* When we see first tuple, create first index page */ + if (state == NULL) + state = _bt_pagestate(wstate, 0); + + if (prev != NULL) /* if this is not the first tuple */ + { + /* is this tuple a duplicate of the previous one? */ + if (_bt_tuples_equal(wstate->index, wstate->inskey, prev, itup, + btspool->nulls_not_distinct)) + { + ItemPointerData tid; + + /* a group starts: look at the tuple that opened it */ + if (!in_group && heap_fetches_left > 0) + { + tid = prev->t_tid; + seen_status = _bt_heap_tuple_status(wstate->heap, &tid); + heap_fetches_left--; + + /* + * The scan saw committed inserts. HOT updates report + * deletes, and the build lock prevents root TID reuse. + */ + Assert(seen_status != HEAPTUPLE_INSERT_IN_PROGRESS); + in_group = true; + } + + /* + * Defer this entry if the budget is exhausted or a + * conflicting tuple is being deleted or updated. + * Validation waits for the writers and inserts surviving + * tuples with a full uniqueness check against entries + * already written. + */ + if (heap_fetches_left == 0 || + seen_status == HEAPTUPLE_DELETE_IN_PROGRESS) + defer_to_validation = true; + else + { + HTSV_Result now_status; + + tid = itup->t_tid; + now_status = _bt_heap_tuple_status(wstate->heap, &tid); + heap_fetches_left--; + Assert(now_status != HEAPTUPLE_INSERT_IN_PROGRESS); + + if (seen_status == HEAPTUPLE_LIVE && + now_status == HEAPTUPLE_LIVE) + _bt_report_duplicate(wstate->index, wstate->heap, itup); + else if (now_status == HEAPTUPLE_DELETE_IN_PROGRESS) + defer_to_validation = true; + + /* + * Keep a live or unresolved member as the reference + * for the group; replace it if it is dead. + */ + if (seen_status == HEAPTUPLE_DEAD) + seen_status = now_status; + } + + if (defer_to_validation) + { + INJECTION_POINT("bt_load_defer_unique_check", NULL); + + /* + * Validation must scan the whole heap to find entries + * skipped here. Any scheme that scans only part of + * the heap would need to track these entries + * separately. + */ + } + } + else + in_group = false; + } + + if (!defer_to_validation) + _bt_buildadd(wstate, state, itup, 0); + else + { + /* + * Deferred entries bypass _bt_buildadd(), so check for + * interrupts and snapshot resets here too. + */ + CHECK_FOR_INTERRUPTS(); + } + + /* + * The tuple is only valid until the next fetch. Keep a copy for + * comparison, reusing the buffer when it is large enough. + */ + if (IndexTupleSize(itup) > prevbufsize) + { + if (prev) + pfree(prev); + prevbufsize = IndexTupleSize(itup); + prev = (IndexTuple) palloc(prevbufsize); + } + memcpy(prev, itup, IndexTupleSize(itup)); + + /* Report progress */ + pgstat_progress_update_param(PROGRESS_CREATEIDX_TUPLES_DONE, + ++tuples_done); + } + if (prev) + pfree(prev); + } + else if (merge) { /* * Another BTSpool for dead tuples exists. Now we have to merge @@ -1413,7 +1560,6 @@ _bt_begin_parallel(BTBuildState *buildstate, bool isconcurrent, int request) WalUsage *walusage; BufferUsage *bufferusage; bool leaderparticipates = true; - bool reset_snapshot; int querylen; #ifdef DISABLE_LEADER_PARTICIPATION @@ -1431,21 +1577,17 @@ _bt_begin_parallel(BTBuildState *buildstate, bool isconcurrent, int request) scantuplesortstates = leaderparticipates ? request + 1 : request; - reset_snapshot = buildstate->spool->reset_snapshot; - /* * Use SnapshotAny for non-concurrent builds so the scan can include - * RECENTLY_DEAD tuples. Concurrent builds that reset snapshots use the - * active snapshot from index_build(), which parallel initialization - * serializes for the workers. Register a fixed MVCC snapshot for - * concurrent unique builds, which do not support resets yet. + * RECENTLY_DEAD tuples. Concurrent builds use the active snapshot from + * index_build(), which parallel initialization serializes for the + * workers. Do not register another copy, as it would pin xmin across + * snapshot resets. */ if (!isconcurrent) snapshot = SnapshotAny; - else if (reset_snapshot) + else snapshot = InvalidSnapshot; - else - snapshot = RegisterSnapshot(GetTransactionSnapshot()); /* * Estimate size for our own PARALLEL_KEY_BTREE_SHARED workspace, and @@ -1457,16 +1599,15 @@ _bt_begin_parallel(BTBuildState *buildstate, bool isconcurrent, int request) shm_toc_estimate_chunk(&pcxt->estimator, estsort); /* - * Unique case requires a second spool, and so we may have to account for - * another shared workspace for that -- PARALLEL_KEY_TUPLESORT_SPOOL2 + * Account for PARALLEL_KEY_TUPLESORT_SPOOL2 if a second spool is needed. */ - if (!btspool->isunique) - shm_toc_estimate_keys(&pcxt->estimator, 2); - else + if (BTBuildNeedsSpool2(btspool->isunique, isconcurrent)) { shm_toc_estimate_chunk(&pcxt->estimator, estsort); shm_toc_estimate_keys(&pcxt->estimator, 3); } + else + shm_toc_estimate_keys(&pcxt->estimator, 2); /* * Estimate space for WalUsage and BufferUsage -- PARALLEL_KEY_WAL_USAGE @@ -1499,8 +1640,6 @@ _bt_begin_parallel(BTBuildState *buildstate, bool isconcurrent, int request) /* If no DSM segment was available, back out (do serial build) */ if (pcxt->seg == NULL) { - if (snapshot != InvalidSnapshot && IsMVCCSnapshot(snapshot)) - UnregisterSnapshot(snapshot); DestroyParallelContext(pcxt); ExitParallelMode(); return; @@ -1513,6 +1652,7 @@ _bt_begin_parallel(BTBuildState *buildstate, bool isconcurrent, int request) btshared->indexrelid = RelationGetRelid(btspool->index); btshared->isunique = btspool->isunique; btshared->nulls_not_distinct = btspool->nulls_not_distinct; + btshared->unique_dead_ignored = btspool->unique_dead_ignored; btshared->isconcurrent = isconcurrent; btshared->scantuplesortstates = scantuplesortstates; btshared->queryid = pgstat_get_my_query_id(); @@ -1527,7 +1667,7 @@ _bt_begin_parallel(BTBuildState *buildstate, bool isconcurrent, int request) table_parallelscan_initialize(btspool->heap, ParallelTableScanFromBTShared(btshared), snapshot, - reset_snapshot); + isconcurrent); /* * Store shared tuplesort-private state, for which we reserved space. @@ -1540,10 +1680,7 @@ _bt_begin_parallel(BTBuildState *buildstate, bool isconcurrent, int request) shm_toc_insert(pcxt->toc, PARALLEL_KEY_BTREE_SHARED, btshared); shm_toc_insert(pcxt->toc, PARALLEL_KEY_TUPLESORT, sharedsort); - /* Unique case requires a second spool, and associated shared state */ - if (!btspool->isunique) - sharedsort2 = NULL; - else + if (BTBuildNeedsSpool2(btspool->isunique, isconcurrent)) { /* * Store additional shared tuplesort-private state, for which we @@ -1556,6 +1693,8 @@ _bt_begin_parallel(BTBuildState *buildstate, bool isconcurrent, int request) shm_toc_insert(pcxt->toc, PARALLEL_KEY_TUPLESORT_SPOOL2, sharedsort2); } + else + sharedsort2 = NULL; /* Store query string for workers */ if (debug_query_string) @@ -1587,7 +1726,6 @@ _bt_begin_parallel(BTBuildState *buildstate, bool isconcurrent, int request) btleader->btshared = btshared; btleader->sharedsort = sharedsort; btleader->sharedsort2 = sharedsort2; - btleader->snapshot = snapshot; btleader->walusage = walusage; btleader->bufferusage = bufferusage; @@ -1607,7 +1745,7 @@ _bt_begin_parallel(BTBuildState *buildstate, bool isconcurrent, int request) * its initial snapshot until every worker has restored it. */ WaitForParallelWorkersToAttach(pcxt); - if (reset_snapshot) + if (isconcurrent) table_parallelscan_wait_restored(ParallelTableScanFromBTShared(btshared), pcxt->nworkers_launched); @@ -1634,9 +1772,6 @@ _bt_end_parallel(BTLeader *btleader) for (i = 0; i < btleader->pcxt->nworkers_launched; i++) InstrAccumParallelQuery(&btleader->bufferusage[i], &btleader->walusage[i]); - /* Free last reference to MVCC snapshot, if one was used */ - if (btleader->snapshot != InvalidSnapshot && IsMVCCSnapshot(btleader->snapshot)) - UnregisterSnapshot(btleader->snapshot); DestroyParallelContext(btleader->pcxt); ExitParallelMode(); } @@ -1714,9 +1849,11 @@ _bt_leader_participate_as_worker(BTBuildState *buildstate) leaderworker->index = buildstate->spool->index; leaderworker->isunique = buildstate->spool->isunique; leaderworker->nulls_not_distinct = buildstate->spool->nulls_not_distinct; + leaderworker->unique_dead_ignored = buildstate->spool->unique_dead_ignored; /* Initialize second spool, if required */ - if (!btleader->btshared->isunique) + if (!BTBuildNeedsSpool2(btleader->btshared->isunique, + btleader->btshared->isconcurrent)) leaderworker2 = NULL; else { @@ -1817,11 +1954,12 @@ _bt_parallel_build_main(dsm_segment *seg, shm_toc *toc) btspool->index = indexRel; btspool->isunique = btshared->isunique; btspool->nulls_not_distinct = btshared->nulls_not_distinct; + btspool->unique_dead_ignored = btshared->unique_dead_ignored; /* Look up shared state private to tuplesort.c */ sharedsort = shm_toc_lookup(toc, PARALLEL_KEY_TUPLESORT, false); tuplesort_attach_shared(sharedsort, seg); - if (!btshared->isunique) + if (!BTBuildNeedsSpool2(btshared->isunique, btshared->isconcurrent)) { btspool2 = NULL; sharedsort2 = NULL; @@ -1901,6 +2039,7 @@ _bt_parallel_scan_and_sort(BTSpool *btspool, BTSpool *btspool2, btspool->index, btspool->isunique, btspool->nulls_not_distinct, + btspool->unique_dead_ignored, sortmem, coordinate, TUPLESORT_NONE); @@ -1923,14 +2062,12 @@ _bt_parallel_scan_and_sort(BTSpool *btspool, BTSpool *btspool2, coordinate2->nParticipants = -1; coordinate2->sharedsort = sharedsort2; btspool2->sortstate = - tuplesort_begin_index_btree(btspool->heap, btspool->index, false, false, + tuplesort_begin_index_btree(btspool->heap, btspool->index, false, false, false, Min(sortmem, work_mem), coordinate2, false); } /* Fill in buildstate for _bt_build_callback() */ - buildstate.isunique = btshared->isunique; - buildstate.nulls_not_distinct = btshared->nulls_not_distinct; buildstate.havedead = false; buildstate.heap = btspool->heap; buildstate.spool = btspool; diff --git a/src/backend/access/nbtree/nbtutils.c b/src/backend/access/nbtree/nbtutils.c index 014faa1622..28e8609db6 100644 --- a/src/backend/access/nbtree/nbtutils.c +++ b/src/backend/access/nbtree/nbtutils.c @@ -20,6 +20,7 @@ #include "access/nbtree.h" #include "access/reloptions.h" #include "access/relscan.h" +#include "access/tableam.h" #include "commands/progress.h" #include "common/int.h" #include "lib/qunique.h" @@ -29,6 +30,7 @@ #include "utils/datum.h" #include "utils/lsyscache.h" #include "utils/rel.h" +#include "utils/snapmgr.h" static int _bt_compare_int(const void *va, const void *vb); @@ -1102,6 +1104,108 @@ _bt_check_natts(Relation rel, bool heapkeyspace, Page page, OffsetNumber offnum) return tupnatts > 0 && tupnatts <= nkeyatts; } +/* + * _bt_heap_tuple_status() -- check a tuple's current state for uniqueness + * + * Equal keys collected under different snapshots may not have been live at + * the same time. Use a dirty snapshot, as _bt_check_unique() does, to check + * their current state. Report in-progress inserts and deletes separately so + * _bt_load() can defer the conflict to validation. + * + * Return HeapTupleSatisfiesVacuum() status codes, but treat all dead tuples + * as HEAPTUPLE_DEAD regardless of whether they can be removed. Check the HOT + * chain and update *tid to the matching version, if any. + */ +HTSV_Result +_bt_heap_tuple_status(Relation heapRel, ItemPointer tid) +{ + SnapshotData snapshot; + + InitDirtySnapshot(snapshot); + + if (!table_fetch_tid(heapRel, tid, &snapshot, NULL)) + return HEAPTUPLE_DEAD; + + if (TransactionIdIsValid(snapshot.xmin)) + return HEAPTUPLE_INSERT_IN_PROGRESS; + if (TransactionIdIsValid(snapshot.xmax)) + return HEAPTUPLE_DELETE_IN_PROGRESS; + + return HEAPTUPLE_LIVE; +} + +/* + * _bt_tuples_equal() -- compare index keys for a uniqueness check + * + * Use the same comparisons as suffix truncation. NULL key values are + * distinct unless the index specifies NULLS NOT DISTINCT. + */ +bool +_bt_tuples_equal(Relation rel, BTScanInsert itup_key, + IndexTuple itup1, IndexTuple itup2, bool nulls_not_distinct) +{ + int keysz = IndexRelationGetNumberOfKeyAttributes(rel); + + /* only a build calls this, and builds write version 4 indexes */ + Assert(itup_key->heapkeyspace); + + /* + * The tuple header bit accounts for all columns, so with INCLUDE columns + * present the key columns have to be checked explicitly. + */ + if (!nulls_not_distinct && + (IndexTupleHasNulls(itup1) || IndexTupleHasNulls(itup2))) + { + TupleDesc itupdesc = RelationGetDescr(rel); + + if (keysz == IndexRelationGetNumberOfAttributes(rel)) + return false; + + for (int attnum = 1; attnum <= keysz; attnum++) + { + bool isNull; + + (void) index_getattr(itup1, attnum, itupdesc, &isNull); + if (isNull) + return false; + (void) index_getattr(itup2, attnum, itupdesc, &isNull); + if (isNull) + return false; + } + } + + if (itup_key->allequalimage) + return _bt_keep_natts_fast(rel, itup1, itup2) > keysz; + else + return _bt_keep_natts(rel, itup1, itup2, itup_key) > keysz; +} + +/* + * _bt_report_duplicate() -- report a duplicate found during index build + * + * Shared by _bt_load() and the tuplesort comparator's early check. + */ +void +_bt_report_duplicate(Relation indexRel, Relation heapRel, IndexTuple itup) +{ + Datum values[INDEX_MAX_KEYS]; + bool isnull[INDEX_MAX_KEYS]; + char *key_desc; + + index_deform_tuple(itup, RelationGetDescr(indexRel), values, isnull); + + key_desc = BuildIndexValueDescription(indexRel, values, isnull); + + ereport(ERROR, + (errcode(ERRCODE_UNIQUE_VIOLATION), + errmsg("could not create unique index \"%s\"", + RelationGetRelationName(indexRel)), + key_desc ? errdetail("Key %s is duplicated.", key_desc) : + errdetail("Duplicate keys exist."), + errtableconstraint(heapRel, + RelationGetRelationName(indexRel)))); +} + /* * * _bt_check_third_page() -- check whether tuple fits on a btree page at all. diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c index c8b0684dd7..6ccbec8e0e 100644 --- a/src/backend/catalog/index.c +++ b/src/backend/catalog/index.c @@ -3124,14 +3124,11 @@ index_update_stats(Relation rel, * * The index AM must opt in with amcanresetsnapshot. The scan decides whether * to perform resets; see SO_RESET_SNAPSHOT. - * - * Unique builds do not support resets yet. */ static bool index_build_resets_snapshots(Relation indexRelation, IndexInfo *indexInfo) { return indexInfo->ii_Concurrent && - !indexInfo->ii_Unique && indexRelation->rd_indam->amcanresetsnapshot; } @@ -3489,11 +3486,9 @@ IndexCheckExclusion(Relation heapRelation, * if we used HeapTupleSatisfiesVacuum). The build can still miss tuples * added while it is running. * - * That snapshot is not held for the whole scan: a non-unique build replaces - * it with a fresh one every so often (SO_RESET_SNAPSHOT), so that the xmin - * horizon can advance while a large table is scanned. Each page is still - * read under one snapshot, and whatever a later snapshot shows that an - * earlier one did not is picked up by the validation described next. + * With SO_RESET_SNAPSHOT, the build can replace the snapshot between + * pages to let xmin advance. Each page is read under one snapshot; the + * validation phase below finds any live tuples missed by the build. * * Next, we mark the index "indisready" (but still not "indisvalid") and * commit the second transaction and start a third. Again we wait for all diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b5b0fe2747..e407783d0e 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -1730,12 +1730,12 @@ DefineIndex(ParseState *pstate, * chains can be created where the new tuple and the old tuple in the * chain have different index keys. * - * We build the index using all tuples that are visible using single or - * multiple refreshing snapshots. We can be sure that any HOT updates to - * these tuples will be compatible with the index, since any updates made - * by transactions that didn't know about the index are now committed or - * rolled back. Thus, each visible tuple is either the end of its - * HOT-chain or the extension of the chain is HOT-safe for this index. + * We build the index from tuples visible to the scan, which may replace + * its snapshot periodically. Any HOT updates to these tuples will be + * compatible with the index: transactions that did not know about the + * index have now committed or rolled back. Each visible tuple is either + * the end of its HOT-chain or followed only by updates that are HOT-safe + * for this index. */ /* Perform concurrent build of index */ diff --git a/src/backend/utils/misc/guc_parameters.dat b/src/backend/utils/misc/guc_parameters.dat index b9d1d925dd..c25cf4a257 100644 --- a/src/backend/utils/misc/guc_parameters.dat +++ b/src/backend/utils/misc/guc_parameters.dat @@ -1356,6 +1356,16 @@ show_hook => 'show_in_hot_standby', }, +{ name => 'index_build_duplicate_check_scale_factor', type => 'real', context => 'PGC_SUSET', group => 'DEVELOPER_OPTIONS', + short_desc => 'Limits heap fetches for early duplicate checks in concurrent unique index builds.', + long_desc => 'Sorting and loading have separate heap-fetch budgets, each a fraction of their input tuple count with a minimum of 1000 fetches. Zero leaves duplicate checks to validation.', + flags => 'GUC_NOT_IN_SAMPLE', + variable => 'index_build_duplicate_check_scale_factor', + boot_val => '0.05', + min => '0.0', + max => '1.0', +}, + { name => 'integer_datetimes', type => 'bool', context => 'PGC_INTERNAL', group => 'PRESET_OPTIONS', short_desc => 'Shows whether datetimes are integer based.', flags => 'GUC_REPORT | GUC_NOT_IN_SAMPLE | GUC_DISALLOW_IN_FILE', diff --git a/src/backend/utils/sort/tuplesort.c b/src/backend/utils/sort/tuplesort.c index 37c40763ee..6e8b5ea7cc 100644 --- a/src/backend/utils/sort/tuplesort.c +++ b/src/backend/utils/sort/tuplesort.c @@ -686,6 +686,7 @@ tuplesort_begin_batch(Tuplesortstate *state) state->tapeset = NULL; state->memtupcount = 0; + state->base.ntuples = 0; state->growmemtuples = true; state->slabAllocatorUsed = false; @@ -1071,6 +1072,7 @@ tuplesort_puttuple_common(Tuplesortstate *state, SortTuple *tuple, /* account for the memory used for this tuple */ USEMEM(state, tuplen); state->tupleMem += tuplen; + state->base.ntuples++; if (!useAbbrev) { diff --git a/src/backend/utils/sort/tuplesortvariants.c b/src/backend/utils/sort/tuplesortvariants.c index c648c78455..4dfd6c57dc 100644 --- a/src/backend/utils/sort/tuplesortvariants.c +++ b/src/backend/utils/sort/tuplesortvariants.c @@ -28,7 +28,9 @@ #include "catalog/index.h" #include "catalog/pg_collation.h" #include "executor/executor.h" +#include "miscadmin.h" #include "pg_trace.h" +#include "storage/proc.h" #include "utils/builtins.h" #include "utils/datum.h" #include "utils/guc.h" @@ -36,6 +38,7 @@ #include "utils/rel.h" #include "utils/tuplesort.h" +double index_build_duplicate_check_scale_factor = 0.05; /* sort-type codes for sort__start probes */ #define HEAP_SORT 0 @@ -136,6 +139,8 @@ typedef struct bool enforceUnique; /* complain if we find duplicate tuples */ bool uniqueNullsNotDistinct; /* unique constraint null treatment */ + bool uniqueDeadIgnored; /* ignore dead tuples in unique check */ + int64 heapFetchCount; /* heap fetches performed in comparator */ } TuplesortIndexBTreeArg; /* @@ -360,6 +365,7 @@ tuplesort_begin_index_btree(Relation heapRel, Relation indexRel, bool enforceUnique, bool uniqueNullsNotDistinct, + bool uniqueDeadIgnored, int workMem, SortCoordinate coordinate, int sortopt) @@ -402,6 +408,8 @@ tuplesort_begin_index_btree(Relation heapRel, arg->index.indexRel = indexRel; arg->enforceUnique = enforceUnique; arg->uniqueNullsNotDistinct = uniqueNullsNotDistinct; + arg->uniqueDeadIgnored = uniqueDeadIgnored; + arg->heapFetchCount = 0; indexScanKey = _bt_mkscankey(indexRel, NULL); @@ -1673,9 +1681,7 @@ comparetup_index_btree_tiebreak(const SortTuple *a, const SortTuple *b, */ if (arg->enforceUnique && !(!arg->uniqueNullsNotDistinct && equal_hasnull)) { - Datum values[INDEX_MAX_KEYS]; - bool isnull[INDEX_MAX_KEYS]; - char *key_desc; + bool uniqueCheckFail = true; /* * Some rather brain-dead implementations of qsort (such as the one in @@ -1685,18 +1691,73 @@ comparetup_index_btree_tiebreak(const SortTuple *a, const SortTuple *b, */ Assert(tuple1 != tuple2); - index_deform_tuple(tuple1, tupDes, values, isnull); - - key_desc = BuildIndexValueDescription(arg->index.indexRel, values, isnull); - - ereport(ERROR, - (errcode(ERRCODE_UNIQUE_VIOLATION), - errmsg("could not create unique index \"%s\"", - RelationGetRelationName(arg->index.indexRel)), - key_desc ? errdetail("Key %s is duplicated.", key_desc) : - errdetail("Duplicate keys exist."), - errtableconstraint(arg->index.heapRel, - RelationGetRelationName(arg->index.indexRel)))); + /* + * Check the heap before reporting a duplicate: either tuple may have + * died since the scan saw it. Limit these fetches with + * index_build_duplicate_check_scale_factor to avoid excessive I/O. + * _bt_load() can check duplicates again within its own budget; + * remaining candidates are left for validation. + */ + if (arg->uniqueDeadIgnored) + { + /* + * Recompute the limit as a fraction of tuples received so far, + * since external sorts compare tuples before input is complete. + * Allow at least 1000 fetches unless early checks are disabled. + * A parallel leader reads worker tapes instead of receiving + * tuples, so it uses the minimum budget. + */ + int64 fetch_limit = base->ntuples * + index_build_duplicate_check_scale_factor; + + if (index_build_duplicate_check_scale_factor > 0) + fetch_limit = Max(fetch_limit, 1000); + + if (arg->heapFetchCount < fetch_limit) + { + bool any_tuple_dead; + ItemPointerData tid = tuple1->t_tid; +#ifdef USE_ASSERT_CHECKING + TransactionId xmin_before = MyProc->xmin; +#endif + + any_tuple_dead = _bt_heap_tuple_status(arg->index.heapRel, + &tid) != HEAPTUPLE_LIVE; + arg->heapFetchCount++; + + if (!any_tuple_dead) + { + tid = tuple2->t_tid; + any_tuple_dead = _bt_heap_tuple_status(arg->index.heapRel, + &tid) != HEAPTUPLE_LIVE; + arg->heapFetchCount++; + } + + if (any_tuple_dead) + { + elog(DEBUG5, "skipping duplicate values because some of them are dead: (%u,%u) vs (%u,%u)", + ItemPointerGetBlockNumber(&tuple1->t_tid), + ItemPointerGetOffsetNumber(&tuple1->t_tid), + ItemPointerGetBlockNumber(&tuple2->t_tid), + ItemPointerGetOffsetNumber(&tuple2->t_tid)); + + uniqueCheckFail = false; + } + + /* + * Heap fetches must not change xmin. It may already be valid + * when an external sort compares tuples during the heap scan. + */ + Assert(MyProc->xmin == xmin_before); + } + else + { + /* Budget exhausted; defer to _bt_load() or validation. */ + uniqueCheckFail = false; + } + } + if (uniqueCheckFail) + _bt_report_duplicate(arg->index.indexRel, arg->index.heapRel, tuple1); } /* diff --git a/src/include/access/nbtree.h b/src/include/access/nbtree.h index 3097e9bb1a..0a8e7f0722 100644 --- a/src/include/access/nbtree.h +++ b/src/include/access/nbtree.h @@ -15,6 +15,7 @@ #define NBTREE_H #include "access/amapi.h" +#include "access/heapam.h" #include "access/itup.h" #include "access/sdir.h" #include "catalog/pg_am_d.h" @@ -1314,6 +1315,13 @@ extern bool _bt_check_natts(Relation rel, bool heapkeyspace, Page page, extern void _bt_check_third_page(Relation rel, Relation heap, bool needheaptidspace, Page page, IndexTuple newtup); extern bool _bt_allequalimage(Relation rel, bool debugmessage); +extern HTSV_Result _bt_heap_tuple_status(Relation heapRel, + ItemPointer tid); +extern bool _bt_tuples_equal(Relation rel, BTScanInsert itup_key, + IndexTuple itup1, IndexTuple itup2, + bool nulls_not_distinct); +extern void _bt_report_duplicate(Relation indexRel, Relation heapRel, + IndexTuple itup); /* * prototypes for functions in nbtvalidate.c diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h index 942f13ae5b..2bbef9e9cc 100644 --- a/src/include/miscadmin.h +++ b/src/include/miscadmin.h @@ -273,6 +273,7 @@ extern PGDLLIMPORT double hash_mem_multiplier; extern PGDLLIMPORT int maintenance_work_mem; extern PGDLLIMPORT int max_parallel_maintenance_workers; extern PGDLLIMPORT int concurrent_index_reset_snapshot_interval; +extern PGDLLIMPORT double index_build_duplicate_check_scale_factor; /* * Upper and lower hard limits for the buffer access strategy ring size diff --git a/src/include/utils/tuplesort.h b/src/include/utils/tuplesort.h index da68f45acf..191010c57c 100644 --- a/src/include/utils/tuplesort.h +++ b/src/include/utils/tuplesort.h @@ -216,6 +216,8 @@ typedef struct bool tuples; /* Can SortTuple.tuple ever be set? */ + int64 ntuples; /* total tuples inserted into sort */ + void *arg; /* Specific information for the sort variant */ } TuplesortPublic; @@ -396,6 +398,7 @@ extern Tuplesortstate *tuplesort_begin_index_btree(Relation heapRel, Relation indexRel, bool enforceUnique, bool uniqueNullsNotDistinct, + bool uniqueDeadIgnored, int workMem, SortCoordinate coordinate, int sortopt); extern Tuplesortstate *tuplesort_begin_index_hash(Relation heapRel, diff --git a/src/test/modules/injection_points/Makefile b/src/test/modules/injection_points/Makefile index 362d891be2..448b2f7d9c 100644 --- a/src/test/modules/injection_points/Makefile +++ b/src/test/modules/injection_points/Makefile @@ -20,6 +20,7 @@ REGRESS_OPTS = --dlpath=$(top_builddir)/src/test/regress WAIT_CLEANUP_TST = $(if $(filter win32,$(PORTNAME)),,wait_cleanup) ISOLATION = basic \ + cic_reset_unique \ inplace \ reindex_concurrently_deferred \ repack \ diff --git a/src/test/modules/injection_points/expected/cic_reset_snapshots.out b/src/test/modules/injection_points/expected/cic_reset_snapshots.out index 78b6d56c35..f1dde577cc 100644 --- a/src/test/modules/injection_points/expected/cic_reset_snapshots.out +++ b/src/test/modules/injection_points/expected/cic_reset_snapshots.out @@ -31,6 +31,14 @@ SELECT injection_points_attach('table_parallelscan_initialize_reset_snapshot', ' (1 row) +-- No concurrent deletes occur here. Deferral is only expected in the +-- tests below that deliberately exhaust the heap-fetch budget. +SELECT injection_points_attach('bt_load_defer_unique_check', 'error'); + injection_points_attach +------------------------- + +(1 row) + CREATE SCHEMA cic_reset_snap; -- Pad rows to fit exactly seven per page at any block size. A scan of -- 35 rows then resets its snapshot five times with a zero interval. @@ -76,7 +84,9 @@ END; $$; ---------------- ALTER TABLE cic_reset_snap.tbl SET (parallel_workers=0); CREATE UNIQUE INDEX CONCURRENTLY idx ON cic_reset_snap.tbl(i); +NOTICE: notice triggered for injection point heap_beginscan_reset_snapshot REINDEX INDEX CONCURRENTLY cic_reset_snap.idx; +NOTICE: notice triggered for injection point heap_beginscan_reset_snapshot DROP INDEX CONCURRENTLY cic_reset_snap.idx; CREATE INDEX CONCURRENTLY idx ON cic_reset_snap.tbl(i); NOTICE: notice triggered for injection point heap_beginscan_reset_snapshot @@ -160,7 +170,9 @@ DROP INDEX CONCURRENTLY cic_reset_snap.idx; -- The same in parallel mode ALTER TABLE cic_reset_snap.tbl SET (parallel_workers=2); CREATE UNIQUE INDEX CONCURRENTLY idx ON cic_reset_snap.tbl(i); +NOTICE: notice triggered for injection point table_parallelscan_initialize_reset_snapshot REINDEX INDEX CONCURRENTLY cic_reset_snap.idx; +NOTICE: notice triggered for injection point table_parallelscan_initialize_reset_snapshot DROP INDEX CONCURRENTLY cic_reset_snap.idx; CREATE INDEX CONCURRENTLY idx ON cic_reset_snap.tbl(i); NOTICE: notice triggered for injection point table_parallelscan_initialize_reset_snapshot @@ -252,12 +264,190 @@ SELECT injection_points_detach('heap_reset_scan_snapshot_swap'); (1 row) +-- Check that each duplicate-detection path reports a uniqueness violation. +-- Drop the invalid index left by each failed build before the next test. +CREATE TABLE cic_reset_snap.dup(i int, j int); +ALTER TABLE cic_reset_snap.dup SET (parallel_workers=0); +INSERT INTO cic_reset_snap.dup SELECT i, i FROM generate_series(1, 1500) s(i); +INSERT INTO cic_reset_snap.dup VALUES (1500, 1500); +INSERT INTO cic_reset_snap.dup SELECT i, i FROM generate_series(1501, 3000) s(i); +-- The fail-fast check in the tuplesort comparator, in-memory sort. +CREATE UNIQUE INDEX CONCURRENTLY dup_idx ON cic_reset_snap.dup(i); +NOTICE: notice triggered for injection point heap_beginscan_reset_snapshot +ERROR: could not create unique index "dup_idx" +DETAIL: Key (i)=(1500) is duplicated. +DROP INDEX cic_reset_snap.dup_idx; +-- The same with sort runs dumped (and compared) while the heap scan is +-- still in progress. +SET maintenance_work_mem = '64kB'; +CREATE UNIQUE INDEX CONCURRENTLY dup_idx ON cic_reset_snap.dup(i); +NOTICE: notice triggered for injection point heap_beginscan_reset_snapshot +ERROR: could not create unique index "dup_idx" +DETAIL: Key (i)=(1500) is duplicated. +RESET maintenance_work_mem; +DROP INDEX cic_reset_snap.dup_idx; +-- With both budgets disabled, _bt_load() skips the duplicate and validation +-- reports it. The notice confirms that the entry was left for validation. +SELECT injection_points_detach('bt_load_defer_unique_check'); + injection_points_detach +------------------------- + +(1 row) + +SELECT injection_points_attach('bt_load_defer_unique_check', 'notice'); + injection_points_attach +------------------------- + +(1 row) + +SET index_build_duplicate_check_scale_factor = 0; +CREATE UNIQUE INDEX CONCURRENTLY dup_idx ON cic_reset_snap.dup(i); +NOTICE: notice triggered for injection point heap_beginscan_reset_snapshot +NOTICE: notice triggered for injection point bt_load_defer_unique_check +ERROR: duplicate key value violates unique constraint "dup_idx" +DETAIL: Key (i)=(1500) already exists. +RESET index_build_duplicate_check_scale_factor; +DROP INDEX cic_reset_snap.dup_idx; +-- INCLUDE columns switch tuple comparison to the per-column path. +CREATE UNIQUE INDEX CONCURRENTLY dup_idx ON cic_reset_snap.dup(i) INCLUDE (j); +NOTICE: notice triggered for injection point heap_beginscan_reset_snapshot +ERROR: could not create unique index "dup_idx" +DETAIL: Key (i)=(1500) is duplicated. +DROP INDEX cic_reset_snap.dup_idx; +-- Either a worker or the leader may report the error, so hide the CONTEXT line. +ALTER TABLE cic_reset_snap.dup SET (parallel_workers=2); +\set VERBOSITY terse +CREATE UNIQUE INDEX CONCURRENTLY dup_idx ON cic_reset_snap.dup(i); +NOTICE: notice triggered for injection point table_parallelscan_initialize_reset_snapshot +ERROR: could not create unique index "dup_idx" +\set VERBOSITY default +DROP INDEX cic_reset_snap.dup_idx; +-- The parallel leader must also leave unchecked duplicates for validation. +SET index_build_duplicate_check_scale_factor = 0; +CREATE UNIQUE INDEX CONCURRENTLY dup_idx ON cic_reset_snap.dup(i); +NOTICE: notice triggered for injection point table_parallelscan_initialize_reset_snapshot +NOTICE: notice triggered for injection point bt_load_defer_unique_check +ERROR: duplicate key value violates unique constraint "dup_idx" +DETAIL: Key (i)=(1500) already exists. +RESET index_build_duplicate_check_scale_factor; +DROP TABLE cic_reset_snap.dup; +-- With no heap-fetch budget, ordinary NULLs must still be written; only +-- NULLS NOT DISTINCT may defer them to validation and report a conflict. +SET index_build_duplicate_check_scale_factor = 0; +CREATE TABLE cic_reset_snap.dupnull(i int); +ALTER TABLE cic_reset_snap.dupnull SET (parallel_workers=0); +INSERT INTO cic_reset_snap.dupnull SELECT g FROM generate_series(1, 3000) g; +INSERT INTO cic_reset_snap.dupnull VALUES (NULL), (NULL); +CREATE UNIQUE INDEX CONCURRENTLY dupnull_idx ON cic_reset_snap.dupnull(i); +NOTICE: notice triggered for injection point heap_beginscan_reset_snapshot +DROP INDEX CONCURRENTLY cic_reset_snap.dupnull_idx; +CREATE UNIQUE INDEX CONCURRENTLY dupnull_idx ON cic_reset_snap.dupnull(i) NULLS NOT DISTINCT; +NOTICE: notice triggered for injection point heap_beginscan_reset_snapshot +NOTICE: notice triggered for injection point bt_load_defer_unique_check +ERROR: duplicate key value violates unique constraint "dupnull_idx" +DETAIL: Key (i)=(null) already exists. +DROP TABLE cic_reset_snap.dupnull; +RESET index_build_duplicate_check_scale_factor; +-- numeric values 1500 and 1500.000 are equal despite different binary +-- representations. With no heap-fetch budget, _bt_load() must recognize +-- the equal keys and leave the second entry for validation. +CREATE TABLE cic_reset_snap.dupn(n numeric); +ALTER TABLE cic_reset_snap.dupn SET (parallel_workers=0); +INSERT INTO cic_reset_snap.dupn SELECT g FROM generate_series(1, 1500) g; +INSERT INTO cic_reset_snap.dupn VALUES (1500.000); +INSERT INTO cic_reset_snap.dupn SELECT g FROM generate_series(1501, 3000) g; +SET index_build_duplicate_check_scale_factor = 0; +CREATE UNIQUE INDEX CONCURRENTLY dupn_idx ON cic_reset_snap.dupn(n); +NOTICE: notice triggered for injection point heap_beginscan_reset_snapshot +NOTICE: notice triggered for injection point bt_load_defer_unique_check +ERROR: duplicate key value violates unique constraint "dupn_idx" +DETAIL: Key (n)=(1500.000) already exists. +RESET index_build_duplicate_check_scale_factor; +DROP TABLE cic_reset_snap.dupn; +SELECT injection_points_detach('bt_load_defer_unique_check'); + injection_points_detach +------------------------- + +(1 row) + +SELECT injection_points_attach('bt_load_defer_unique_check', 'error'); + injection_points_attach +------------------------- + +(1 row) + +-- Comparing enum values added with BEFORE/AFTER can acquire a catalog +-- snapshot on a syscache miss. Force misses with debug_discard_caches, +-- where available, to test this during snapshot resets. +CREATE TYPE cic_reset_snap.rainbow AS ENUM ('red','green','blue'); +ALTER TYPE cic_reset_snap.rainbow ADD VALUE 'teal' BEFORE 'green'; +CREATE TABLE cic_reset_snap.enum_tbl(c cic_reset_snap.rainbow); +ALTER TABLE cic_reset_snap.enum_tbl SET (parallel_workers=0); +INSERT INTO cic_reset_snap.enum_tbl + SELECT (CASE WHEN g % 2 = 0 THEN 'teal' ELSE 'blue' END)::cic_reset_snap.rainbow + FROM generate_series(1, 200) g; +SELECT (max_val::int > 0) AS discard_caches_enabled + FROM pg_settings WHERE name = 'debug_discard_caches' \gset +\if :discard_caches_enabled +SET debug_discard_caches = 1; +\endif +CREATE INDEX CONCURRENTLY enum_idx ON cic_reset_snap.enum_tbl(c); +NOTICE: notice triggered for injection point heap_beginscan_reset_snapshot +RESET debug_discard_caches; +DROP TABLE cic_reset_snap.enum_tbl; +-- A cursor left open by a support function retains the scan snapshot. +-- The build must tolerate this, even though it keeps xmin pinned. +CREATE FUNCTION cic_reset_snap.leak_cursor(i int, at int) RETURNS bool IMMUTABLE +LANGUAGE plpgsql AS $$ +DECLARE c refcursor := 'cic_leaked_cursor_' || at; +BEGIN + IF i = at THEN OPEN c FOR SELECT 1; END IF; + RETURN true; +END; $$; +-- Report the horizon pinned by the cursor instead of raising an error. +SELECT injection_points_detach('heap_reset_scan_snapshot_xmin_pinned'); + injection_points_detach +------------------------- + +(1 row) + +SELECT injection_points_attach('heap_reset_scan_snapshot_xmin_pinned', 'notice'); + injection_points_attach +------------------------- + +(1 row) + +-- No reset occurs after this small scan opens the cursor. +CREATE TABLE cic_reset_snap.cur_small(i int); +ALTER TABLE cic_reset_snap.cur_small SET (parallel_workers=0); +INSERT INTO cic_reset_snap.cur_small SELECT g FROM generate_series(1, 10) g; +CREATE INDEX CONCURRENTLY cur_small_idx ON cic_reset_snap.cur_small(i) + WHERE cic_reset_snap.leak_cursor(i, 5); +NOTICE: notice triggered for injection point heap_beginscan_reset_snapshot +DROP TABLE cic_reset_snap.cur_small; +-- The larger table reaches another reset after opening the cursor. +-- Disable the notice: its count depends on the page layout. +SELECT injection_points_detach('heap_reset_scan_snapshot_xmin_pinned'); + injection_points_detach +------------------------- + +(1 row) + +CREATE TABLE cic_reset_snap.cur_tbl(i int); +ALTER TABLE cic_reset_snap.cur_tbl SET (parallel_workers=0); +INSERT INTO cic_reset_snap.cur_tbl SELECT g FROM generate_series(1, 5000) g; +CREATE INDEX CONCURRENTLY cur_idx ON cic_reset_snap.cur_tbl(i) + WHERE cic_reset_snap.leak_cursor(i, 3000); +NOTICE: notice triggered for injection point heap_beginscan_reset_snapshot +DROP TABLE cic_reset_snap.cur_tbl; DROP SCHEMA cic_reset_snap CASCADE; -NOTICE: drop cascades to 4 other objects +NOTICE: drop cascades to 6 other objects DETAIL: drop cascades to table cic_reset_snap.tbl drop cascades to function cic_reset_snap.predicate_stable(integer) drop cascades to function cic_reset_snap.predicate_stable_no_param() drop cascades to operator family cic_reset_snap.int4_bloom_storage_ops for access method brin +drop cascades to type cic_reset_snap.rainbow +drop cascades to function cic_reset_snap.leak_cursor(integer,integer) SELECT injection_points_detach('heap_beginscan_reset_snapshot'); injection_points_detach ------------------------- @@ -270,4 +460,10 @@ SELECT injection_points_detach('table_parallelscan_initialize_reset_snapshot'); (1 row) +SELECT injection_points_detach('bt_load_defer_unique_check'); + injection_points_detach +------------------------- + +(1 row) + DROP EXTENSION injection_points; diff --git a/src/test/modules/injection_points/expected/cic_reset_unique.out b/src/test/modules/injection_points/expected/cic_reset_unique.out new file mode 100644 index 0000000000..fe3425bc84 --- /dev/null +++ b/src/test/modules/injection_points/expected/cic_reset_unique.out @@ -0,0 +1,381 @@ +Parsed test spec with 3 sessions + +starting permutation: s3_vacuum build s3_wake s2_begin s2_delete s3_insert s3_scan_end s3_await_defer s2_commit s3_build_end +step s3_vacuum: VACUUM cic_uniq; +step build: CREATE UNIQUE INDEX CONCURRENTLY cic_uniq_idx ON cic_uniq(k); +step s3_wake: SELECT injection_points_wakeup('heap_reset_scan_snapshot_swap'); +injection_points_wakeup +----------------------- + +(1 row) + +step s2_begin: BEGIN; +step s2_delete: DELETE FROM cic_uniq WHERE k = 1; +step s3_insert: INSERT INTO cic_uniq VALUES (1, 'y'); +step s3_scan_end: + SELECT injection_points_detach('heap_reset_scan_snapshot_swap'); + SELECT injection_points_wakeup('heap_reset_scan_snapshot_swap'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step s3_await_defer: + DO $$ + DECLARE tries int := 0; + BEGIN + LOOP + PERFORM 1 FROM pg_stat_activity + WHERE wait_event = 'bt_load_defer_unique_check'; + EXIT WHEN FOUND; + IF tries >= 1200 THEN + RAISE 'the build did not reach the deferral point'; + END IF; + PERFORM pg_sleep(0.05); + tries := tries + 1; + END LOOP; + END $$; + +step s2_commit: COMMIT; +step s3_build_end: + SELECT injection_points_detach('bt_load_defer_unique_check'); + SELECT injection_points_wakeup('bt_load_defer_unique_check'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step build: <... completed> + +starting permutation: s3_vacuum build s3_wake s2_begin s2_delete s3_insert s3_scan_end s3_await_defer s2_rollback s3_build_end +step s3_vacuum: VACUUM cic_uniq; +step build: CREATE UNIQUE INDEX CONCURRENTLY cic_uniq_idx ON cic_uniq(k); +step s3_wake: SELECT injection_points_wakeup('heap_reset_scan_snapshot_swap'); +injection_points_wakeup +----------------------- + +(1 row) + +step s2_begin: BEGIN; +step s2_delete: DELETE FROM cic_uniq WHERE k = 1; +step s3_insert: INSERT INTO cic_uniq VALUES (1, 'y'); +step s3_scan_end: + SELECT injection_points_detach('heap_reset_scan_snapshot_swap'); + SELECT injection_points_wakeup('heap_reset_scan_snapshot_swap'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step s3_await_defer: + DO $$ + DECLARE tries int := 0; + BEGIN + LOOP + PERFORM 1 FROM pg_stat_activity + WHERE wait_event = 'bt_load_defer_unique_check'; + EXIT WHEN FOUND; + IF tries >= 1200 THEN + RAISE 'the build did not reach the deferral point'; + END IF; + PERFORM pg_sleep(0.05); + tries := tries + 1; + END LOOP; + END $$; + +step s2_rollback: ROLLBACK; +step s3_build_end: + SELECT injection_points_detach('bt_load_defer_unique_check'); + SELECT injection_points_wakeup('bt_load_defer_unique_check'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step build: <... completed> +ERROR: duplicate key value violates unique constraint "cic_uniq_idx" + +starting permutation: s3_free_hot_slot s3_vacuum build s3_wake s2_begin s2_hot_update s3_insert s3_scan_end s3_await_defer s2_commit s3_build_end +step s3_free_hot_slot: DELETE FROM cic_uniq WHERE k = 2; +step s3_vacuum: VACUUM cic_uniq; +step build: CREATE UNIQUE INDEX CONCURRENTLY cic_uniq_idx ON cic_uniq(k); +step s3_wake: SELECT injection_points_wakeup('heap_reset_scan_snapshot_swap'); +injection_points_wakeup +----------------------- + +(1 row) + +step s2_begin: BEGIN; +step s2_hot_update: + UPDATE cic_uniq SET pad = overlay(pad placing 'y' from 1 for 1) WHERE k = 1; + SELECT n_tup_upd, n_tup_hot_upd FROM pg_stat_xact_user_tables + WHERE relid = 'cic_uniq'::regclass; + +n_tup_upd|n_tup_hot_upd +---------+------------- + 1| 1 +(1 row) + +step s3_insert: INSERT INTO cic_uniq VALUES (1, 'y'); +step s3_scan_end: + SELECT injection_points_detach('heap_reset_scan_snapshot_swap'); + SELECT injection_points_wakeup('heap_reset_scan_snapshot_swap'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step s3_await_defer: + DO $$ + DECLARE tries int := 0; + BEGIN + LOOP + PERFORM 1 FROM pg_stat_activity + WHERE wait_event = 'bt_load_defer_unique_check'; + EXIT WHEN FOUND; + IF tries >= 1200 THEN + RAISE 'the build did not reach the deferral point'; + END IF; + PERFORM pg_sleep(0.05); + tries := tries + 1; + END LOOP; + END $$; + +step s2_commit: COMMIT; +step s3_build_end: + SELECT injection_points_detach('bt_load_defer_unique_check'); + SELECT injection_points_wakeup('bt_load_defer_unique_check'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step build: <... completed> +ERROR: duplicate key value violates unique constraint "cic_uniq_idx" + +starting permutation: s3_free_hot_slot s3_vacuum build s3_wake s2_begin s2_hot_update s3_insert s3_scan_end s3_await_defer s2_rollback s3_build_end +step s3_free_hot_slot: DELETE FROM cic_uniq WHERE k = 2; +step s3_vacuum: VACUUM cic_uniq; +step build: CREATE UNIQUE INDEX CONCURRENTLY cic_uniq_idx ON cic_uniq(k); +step s3_wake: SELECT injection_points_wakeup('heap_reset_scan_snapshot_swap'); +injection_points_wakeup +----------------------- + +(1 row) + +step s2_begin: BEGIN; +step s2_hot_update: + UPDATE cic_uniq SET pad = overlay(pad placing 'y' from 1 for 1) WHERE k = 1; + SELECT n_tup_upd, n_tup_hot_upd FROM pg_stat_xact_user_tables + WHERE relid = 'cic_uniq'::regclass; + +n_tup_upd|n_tup_hot_upd +---------+------------- + 1| 1 +(1 row) + +step s3_insert: INSERT INTO cic_uniq VALUES (1, 'y'); +step s3_scan_end: + SELECT injection_points_detach('heap_reset_scan_snapshot_swap'); + SELECT injection_points_wakeup('heap_reset_scan_snapshot_swap'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step s3_await_defer: + DO $$ + DECLARE tries int := 0; + BEGIN + LOOP + PERFORM 1 FROM pg_stat_activity + WHERE wait_event = 'bt_load_defer_unique_check'; + EXIT WHEN FOUND; + IF tries >= 1200 THEN + RAISE 'the build did not reach the deferral point'; + END IF; + PERFORM pg_sleep(0.05); + tries := tries + 1; + END LOOP; + END $$; + +step s2_rollback: ROLLBACK; +step s3_build_end: + SELECT injection_points_detach('bt_load_defer_unique_check'); + SELECT injection_points_wakeup('bt_load_defer_unique_check'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step build: <... completed> +ERROR: duplicate key value violates unique constraint "cic_uniq_idx" + +starting permutation: zero_budget s3_vacuum build s3_wake s2_begin s2_delete s3_insert s3_scan_end s3_await_defer s2_commit s3_build_end +step zero_budget: SET index_build_duplicate_check_scale_factor = 0; +step s3_vacuum: VACUUM cic_uniq; +step build: CREATE UNIQUE INDEX CONCURRENTLY cic_uniq_idx ON cic_uniq(k); +step s3_wake: SELECT injection_points_wakeup('heap_reset_scan_snapshot_swap'); +injection_points_wakeup +----------------------- + +(1 row) + +step s2_begin: BEGIN; +step s2_delete: DELETE FROM cic_uniq WHERE k = 1; +step s3_insert: INSERT INTO cic_uniq VALUES (1, 'y'); +step s3_scan_end: + SELECT injection_points_detach('heap_reset_scan_snapshot_swap'); + SELECT injection_points_wakeup('heap_reset_scan_snapshot_swap'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step s3_await_defer: + DO $$ + DECLARE tries int := 0; + BEGIN + LOOP + PERFORM 1 FROM pg_stat_activity + WHERE wait_event = 'bt_load_defer_unique_check'; + EXIT WHEN FOUND; + IF tries >= 1200 THEN + RAISE 'the build did not reach the deferral point'; + END IF; + PERFORM pg_sleep(0.05); + tries := tries + 1; + END LOOP; + END $$; + +step s2_commit: COMMIT; +step s3_build_end: + SELECT injection_points_detach('bt_load_defer_unique_check'); + SELECT injection_points_wakeup('bt_load_defer_unique_check'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step build: <... completed> + +starting permutation: zero_budget s3_vacuum build s3_wake s2_begin s2_delete s3_insert s3_scan_end s3_await_defer s2_rollback s3_build_end +step zero_budget: SET index_build_duplicate_check_scale_factor = 0; +step s3_vacuum: VACUUM cic_uniq; +step build: CREATE UNIQUE INDEX CONCURRENTLY cic_uniq_idx ON cic_uniq(k); +step s3_wake: SELECT injection_points_wakeup('heap_reset_scan_snapshot_swap'); +injection_points_wakeup +----------------------- + +(1 row) + +step s2_begin: BEGIN; +step s2_delete: DELETE FROM cic_uniq WHERE k = 1; +step s3_insert: INSERT INTO cic_uniq VALUES (1, 'y'); +step s3_scan_end: + SELECT injection_points_detach('heap_reset_scan_snapshot_swap'); + SELECT injection_points_wakeup('heap_reset_scan_snapshot_swap'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step s3_await_defer: + DO $$ + DECLARE tries int := 0; + BEGIN + LOOP + PERFORM 1 FROM pg_stat_activity + WHERE wait_event = 'bt_load_defer_unique_check'; + EXIT WHEN FOUND; + IF tries >= 1200 THEN + RAISE 'the build did not reach the deferral point'; + END IF; + PERFORM pg_sleep(0.05); + tries := tries + 1; + END LOOP; + END $$; + +step s2_rollback: ROLLBACK; +step s3_build_end: + SELECT injection_points_detach('bt_load_defer_unique_check'); + SELECT injection_points_wakeup('bt_load_defer_unique_check'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step build: <... completed> +ERROR: duplicate key value violates unique constraint "cic_uniq_idx" diff --git a/src/test/modules/injection_points/meson.build b/src/test/modules/injection_points/meson.build index 0800f4e444..f22f1842be 100644 --- a/src/test/modules/injection_points/meson.build +++ b/src/test/modules/injection_points/meson.build @@ -28,6 +28,7 @@ test_install_data += files( injection_points_isolation = [ 'basic', + 'cic_reset_unique', 'inplace', 'reindex_concurrently_deferred', 'repack', diff --git a/src/test/modules/injection_points/specs/cic_reset_unique.spec b/src/test/modules/injection_points/specs/cic_reset_unique.spec new file mode 100644 index 0000000000..55a39a3617 --- /dev/null +++ b/src/test/modules/injection_points/specs/cic_reset_unique.spec @@ -0,0 +1,109 @@ +# Test uniqueness checks across snapshot resets. +# +# The scan collects equal keys under different snapshots while a delete +# remains uncommitted. The build defers the conflict to validation, +# which must succeed if the delete commits and fail if it rolls back. +# An uncommitted HOT update must also defer the check, but leaves duplicates +# whether it commits or rolls back. +# +# Pause before a snapshot reset to insert the second row, then at +# bt_load_defer_unique_check to control when the delete or update finishes. + +setup +{ + CREATE EXTENSION injection_points; + CREATE TABLE cic_uniq (k int, pad text); + ALTER TABLE cic_uniq ALTER COLUMN pad SET STORAGE PLAIN; + ALTER TABLE cic_uniq SET (parallel_workers = 0); + -- Rows are padded to the largest multiple of 8 that fits 7 times into the + -- space a page has for tuples (header 24, line pointer 4, row header + int + -- + varlena header 32): 7 rows per page whatever the block size, with no + -- room left for another row. Free one slot on a page the scan reaches + -- late, so that the row inserted during the build lands on a page the scan + -- has not read yet. + INSERT INTO cic_uniq + SELECT g, repeat('x', ((current_setting('block_size')::int - 24) / 7 - 4) / 8 * 8 - 32) + FROM generate_series(1, 35) g; + DELETE FROM cic_uniq WHERE k = 25; +} + +teardown +{ + DROP TABLE cic_uniq; + DROP EXTENSION injection_points; +} + +session s1 +setup +{ + SELECT injection_points_set_local(); + SELECT injection_points_attach('heap_reset_scan_snapshot_swap', 'wait'); + SELECT injection_points_attach('bt_load_defer_unique_check', 'wait'); + SET concurrent_index_reset_snapshot_interval = 0; + SET index_build_duplicate_check_scale_factor = 1; +} +step zero_budget { SET index_build_duplicate_check_scale_factor = 0; } +step build { CREATE UNIQUE INDEX CONCURRENTLY cic_uniq_idx ON cic_uniq(k); } + +session s2 +step s2_begin { BEGIN; } +step s2_delete { DELETE FROM cic_uniq WHERE k = 1; } +step s2_hot_update { + UPDATE cic_uniq SET pad = overlay(pad placing 'y' from 1 for 1) WHERE k = 1; + SELECT n_tup_upd, n_tup_hot_upd FROM pg_stat_xact_user_tables + WHERE relid = 'cic_uniq'::regclass; +} +step s2_commit { COMMIT; } +step s2_rollback { ROLLBACK; } + +session s3 +# Make room for a HOT update of the first row before the build starts. +step s3_free_hot_slot { DELETE FROM cic_uniq WHERE k = 2; } +# Record the freed slot in the FSM so the new row stays within the +# scan's block range, which is fixed when the scan starts. +step s3_vacuum { VACUUM cic_uniq; } +step s3_wake { SELECT injection_points_wakeup('heap_reset_scan_snapshot_swap'); } +step s3_insert { INSERT INTO cic_uniq VALUES (1, 'y'); } +# Let the scan run to its end; the build then stops at the deferral point. +step s3_scan_end { + SELECT injection_points_detach('heap_reset_scan_snapshot_swap'); + SELECT injection_points_wakeup('heap_reset_scan_snapshot_swap'); +} +# Wait for the build to defer the uniqueness check before ending the +# deleting or updating transaction. +step s3_await_defer { + DO $$ + DECLARE tries int := 0; + BEGIN + LOOP + PERFORM 1 FROM pg_stat_activity + WHERE wait_event = 'bt_load_defer_unique_check'; + EXIT WHEN FOUND; + IF tries >= 1200 THEN + RAISE 'the build did not reach the deferral point'; + END IF; + PERFORM pg_sleep(0.05); + tries := tries + 1; + END LOOP; + END $$; +} +# Let the build finish and reach the validation phase. +step s3_build_end { + SELECT injection_points_detach('bt_load_defer_unique_check'); + SELECT injection_points_wakeup('bt_load_defer_unique_check'); +} + +# Committing the delete leaves unique data, so the build must succeed. +permutation s3_vacuum build s3_wake s2_begin s2_delete s3_insert s3_scan_end s3_await_defer s2_commit s3_build_end + +# Rolling back the delete leaves duplicates, so the build must fail. +permutation s3_vacuum build s3_wake s2_begin s2_delete s3_insert s3_scan_end s3_await_defer s2_rollback s3_build_end + +# Neither outcome of a HOT update removes the conflicting key. +permutation s3_free_hot_slot s3_vacuum build s3_wake s2_begin s2_hot_update s3_insert s3_scan_end s3_await_defer s2_commit s3_build_end +permutation s3_free_hot_slot s3_vacuum build s3_wake s2_begin s2_hot_update s3_insert s3_scan_end s3_await_defer s2_rollback s3_build_end + +# With no heap fetches, validation must insert the surviving row after a +# committed delete, and reject the duplicates after a rollback. +permutation zero_budget s3_vacuum build s3_wake s2_begin s2_delete s3_insert s3_scan_end s3_await_defer s2_commit s3_build_end +permutation zero_budget s3_vacuum build s3_wake s2_begin s2_delete s3_insert s3_scan_end s3_await_defer s2_rollback s3_build_end diff --git a/src/test/modules/injection_points/sql/cic_reset_snapshots.sql b/src/test/modules/injection_points/sql/cic_reset_snapshots.sql index 23939f5412..5ef597d887 100644 --- a/src/test/modules/injection_points/sql/cic_reset_snapshots.sql +++ b/src/test/modules/injection_points/sql/cic_reset_snapshots.sql @@ -7,6 +7,9 @@ SELECT injection_points_attach('heap_reset_scan_snapshot_xmin_pinned', 'error'); SELECT injection_points_attach('heap_reset_scan_snapshot_xid_assigned', 'notice'); SELECT injection_points_attach('heap_beginscan_reset_snapshot', 'notice'); SELECT injection_points_attach('table_parallelscan_initialize_reset_snapshot', 'notice'); +-- No concurrent deletes occur here. Deferral is only expected in the +-- tests below that deliberately exhaust the heap-fetch budget. +SELECT injection_points_attach('bt_load_defer_unique_check', 'error'); CREATE SCHEMA cic_reset_snap; -- Pad rows to fit exactly seven per page at any block size. A scan of @@ -157,9 +160,135 @@ DROP INDEX CONCURRENTLY cic_reset_snap.idx; RESET default_transaction_isolation; SELECT injection_points_detach('heap_reset_scan_snapshot_swap'); +-- Check that each duplicate-detection path reports a uniqueness violation. +-- Drop the invalid index left by each failed build before the next test. +CREATE TABLE cic_reset_snap.dup(i int, j int); +ALTER TABLE cic_reset_snap.dup SET (parallel_workers=0); +INSERT INTO cic_reset_snap.dup SELECT i, i FROM generate_series(1, 1500) s(i); +INSERT INTO cic_reset_snap.dup VALUES (1500, 1500); +INSERT INTO cic_reset_snap.dup SELECT i, i FROM generate_series(1501, 3000) s(i); + +-- The fail-fast check in the tuplesort comparator, in-memory sort. +CREATE UNIQUE INDEX CONCURRENTLY dup_idx ON cic_reset_snap.dup(i); +DROP INDEX cic_reset_snap.dup_idx; + +-- The same with sort runs dumped (and compared) while the heap scan is +-- still in progress. +SET maintenance_work_mem = '64kB'; +CREATE UNIQUE INDEX CONCURRENTLY dup_idx ON cic_reset_snap.dup(i); +RESET maintenance_work_mem; +DROP INDEX cic_reset_snap.dup_idx; + +-- With both budgets disabled, _bt_load() skips the duplicate and validation +-- reports it. The notice confirms that the entry was left for validation. +SELECT injection_points_detach('bt_load_defer_unique_check'); +SELECT injection_points_attach('bt_load_defer_unique_check', 'notice'); +SET index_build_duplicate_check_scale_factor = 0; +CREATE UNIQUE INDEX CONCURRENTLY dup_idx ON cic_reset_snap.dup(i); +RESET index_build_duplicate_check_scale_factor; +DROP INDEX cic_reset_snap.dup_idx; + +-- INCLUDE columns switch tuple comparison to the per-column path. +CREATE UNIQUE INDEX CONCURRENTLY dup_idx ON cic_reset_snap.dup(i) INCLUDE (j); +DROP INDEX cic_reset_snap.dup_idx; + +-- Either a worker or the leader may report the error, so hide the CONTEXT line. +ALTER TABLE cic_reset_snap.dup SET (parallel_workers=2); +\set VERBOSITY terse +CREATE UNIQUE INDEX CONCURRENTLY dup_idx ON cic_reset_snap.dup(i); +\set VERBOSITY default +DROP INDEX cic_reset_snap.dup_idx; + +-- The parallel leader must also leave unchecked duplicates for validation. +SET index_build_duplicate_check_scale_factor = 0; +CREATE UNIQUE INDEX CONCURRENTLY dup_idx ON cic_reset_snap.dup(i); +RESET index_build_duplicate_check_scale_factor; +DROP TABLE cic_reset_snap.dup; + +-- With no heap-fetch budget, ordinary NULLs must still be written; only +-- NULLS NOT DISTINCT may defer them to validation and report a conflict. +SET index_build_duplicate_check_scale_factor = 0; +CREATE TABLE cic_reset_snap.dupnull(i int); +ALTER TABLE cic_reset_snap.dupnull SET (parallel_workers=0); +INSERT INTO cic_reset_snap.dupnull SELECT g FROM generate_series(1, 3000) g; +INSERT INTO cic_reset_snap.dupnull VALUES (NULL), (NULL); +CREATE UNIQUE INDEX CONCURRENTLY dupnull_idx ON cic_reset_snap.dupnull(i); +DROP INDEX CONCURRENTLY cic_reset_snap.dupnull_idx; +CREATE UNIQUE INDEX CONCURRENTLY dupnull_idx ON cic_reset_snap.dupnull(i) NULLS NOT DISTINCT; +DROP TABLE cic_reset_snap.dupnull; +RESET index_build_duplicate_check_scale_factor; + +-- numeric values 1500 and 1500.000 are equal despite different binary +-- representations. With no heap-fetch budget, _bt_load() must recognize +-- the equal keys and leave the second entry for validation. +CREATE TABLE cic_reset_snap.dupn(n numeric); +ALTER TABLE cic_reset_snap.dupn SET (parallel_workers=0); +INSERT INTO cic_reset_snap.dupn SELECT g FROM generate_series(1, 1500) g; +INSERT INTO cic_reset_snap.dupn VALUES (1500.000); +INSERT INTO cic_reset_snap.dupn SELECT g FROM generate_series(1501, 3000) g; +SET index_build_duplicate_check_scale_factor = 0; +CREATE UNIQUE INDEX CONCURRENTLY dupn_idx ON cic_reset_snap.dupn(n); +RESET index_build_duplicate_check_scale_factor; +DROP TABLE cic_reset_snap.dupn; +SELECT injection_points_detach('bt_load_defer_unique_check'); +SELECT injection_points_attach('bt_load_defer_unique_check', 'error'); + +-- Comparing enum values added with BEFORE/AFTER can acquire a catalog +-- snapshot on a syscache miss. Force misses with debug_discard_caches, +-- where available, to test this during snapshot resets. +CREATE TYPE cic_reset_snap.rainbow AS ENUM ('red','green','blue'); +ALTER TYPE cic_reset_snap.rainbow ADD VALUE 'teal' BEFORE 'green'; +CREATE TABLE cic_reset_snap.enum_tbl(c cic_reset_snap.rainbow); +ALTER TABLE cic_reset_snap.enum_tbl SET (parallel_workers=0); +INSERT INTO cic_reset_snap.enum_tbl + SELECT (CASE WHEN g % 2 = 0 THEN 'teal' ELSE 'blue' END)::cic_reset_snap.rainbow + FROM generate_series(1, 200) g; +SELECT (max_val::int > 0) AS discard_caches_enabled + FROM pg_settings WHERE name = 'debug_discard_caches' \gset +\if :discard_caches_enabled +SET debug_discard_caches = 1; +\endif +CREATE INDEX CONCURRENTLY enum_idx ON cic_reset_snap.enum_tbl(c); +RESET debug_discard_caches; +DROP TABLE cic_reset_snap.enum_tbl; + +-- A cursor left open by a support function retains the scan snapshot. +-- The build must tolerate this, even though it keeps xmin pinned. +CREATE FUNCTION cic_reset_snap.leak_cursor(i int, at int) RETURNS bool IMMUTABLE +LANGUAGE plpgsql AS $$ +DECLARE c refcursor := 'cic_leaked_cursor_' || at; +BEGIN + IF i = at THEN OPEN c FOR SELECT 1; END IF; + RETURN true; +END; $$; + +-- Report the horizon pinned by the cursor instead of raising an error. +SELECT injection_points_detach('heap_reset_scan_snapshot_xmin_pinned'); +SELECT injection_points_attach('heap_reset_scan_snapshot_xmin_pinned', 'notice'); + +-- No reset occurs after this small scan opens the cursor. +CREATE TABLE cic_reset_snap.cur_small(i int); +ALTER TABLE cic_reset_snap.cur_small SET (parallel_workers=0); +INSERT INTO cic_reset_snap.cur_small SELECT g FROM generate_series(1, 10) g; +CREATE INDEX CONCURRENTLY cur_small_idx ON cic_reset_snap.cur_small(i) + WHERE cic_reset_snap.leak_cursor(i, 5); +DROP TABLE cic_reset_snap.cur_small; + +-- The larger table reaches another reset after opening the cursor. +-- Disable the notice: its count depends on the page layout. +SELECT injection_points_detach('heap_reset_scan_snapshot_xmin_pinned'); + +CREATE TABLE cic_reset_snap.cur_tbl(i int); +ALTER TABLE cic_reset_snap.cur_tbl SET (parallel_workers=0); +INSERT INTO cic_reset_snap.cur_tbl SELECT g FROM generate_series(1, 5000) g; +CREATE INDEX CONCURRENTLY cur_idx ON cic_reset_snap.cur_tbl(i) + WHERE cic_reset_snap.leak_cursor(i, 3000); +DROP TABLE cic_reset_snap.cur_tbl; + DROP SCHEMA cic_reset_snap CASCADE; SELECT injection_points_detach('heap_beginscan_reset_snapshot'); SELECT injection_points_detach('table_parallelscan_initialize_reset_snapshot'); +SELECT injection_points_detach('bt_load_defer_unique_check'); DROP EXTENSION injection_points; -- 2.43.0