From d973939a318ed0d5346c5e11e8fca49f8a1a5e1d Mon Sep 17 00:00:00 2001 From: jian he Date: Tue, 29 Sep 2026 15:21:31 +0800 Subject: [PATCH v16 1/1] using indexscan to speedup add not null constraints This patch tries to use index_beginscan() / table_index_getnext_slot() / index_endscan() mentioned in [1] to speedup adding not-null constraints to the existing table. The main logic happens in phase 3 ATRewriteTable->index_check_notnull 1. collect all not-null constraints. 2. For each not-null constraint, check whether there is a corresponding index available for validation. If not, then cannot use indexscan mechanism to verify not-null constraints. 3. If any of the following conditions are true * table scan or rewrite is required, * table wasn't locked with `AccessExclusiveLock` * the NOT NULL constraint applies to a virtual generated column then we cannot use indexscan to fast vertify column is not null or not. If all conditions are satisfied, attempt to use indexscan to verify whether the column contains any NULL values. concurrency concern: Under AccessExclusiveLock no other transaction can be changing the table. A fresh MVCC snapshot, GetLatestSnapshot(), therefore sees exactly the rows the full table scan would check. Nothing can add or change a row before we commit, so if the index scan finds no NULL, the table scan wouldn't find one either. As with the table scan, rows already deleted but still visible to older snapshots aren't checked. references: [1] https://postgr.es/m/CA%2BTgmoa5NKz8iGW_9v7wz%3D-%2BzQFu%3DE4SZoaTaU1znLaEXRYp-Q%40mail.gmail.com [2] https://postgr.es/m/900056D1-32DF-4927-8251-3E0C0DC407FD%40anarazel.de discussion: https://postgr.es/m/CACJufxFiW=4k1is=F1J=r-Cx1RuByXQPUrWB331U47rSnGz+hw@mail.gmail.com commitfest entry: https://commitfest.postgresql.org/patch/5444 --- src/backend/commands/tablecmds.c | 277 +++++++++++++++++- src/test/modules/test_misc/meson.build | 1 + .../t/016_indexscan_validate_notnull.pl | 145 +++++++++ 3 files changed, 413 insertions(+), 10 deletions(-) create mode 100644 src/test/modules/test_misc/t/016_indexscan_validate_notnull.pl diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index 0274d892f2e..c2a4d402597 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -212,16 +212,19 @@ typedef struct AlteredTableInfo List *changedStatisticsOwners; /* owners of same */ } AlteredTableInfo; -/* Struct describing one new constraint to check in Phase 3 scan */ -/* Note: new not-null constraints are handled elsewhere */ +/* + * Struct describing one new constraint to check in Phase 3 scan. Note: new + * not-null constraints were also added to Phase 3. + */ typedef struct NewConstraint { char *name; /* Constraint name, or NULL if none */ - ConstrType contype; /* CHECK or FOREIGN */ + ConstrType contype; /* CHECK or FOREIGN or NOT NULL */ Oid refrelid; /* PK rel, if FOREIGN */ Oid refindid; /* OID of PK's index, if FOREIGN */ bool conwithperiod; /* Whether the new FOREIGN KEY uses PERIOD */ Oid conid; /* OID of pg_constraint entry, if FOREIGN */ + int attnum; /* NOT NULL constraint attribute number */ Node *qual; /* Check expr or CONSTR_FOREIGN Constraint */ ExprState *qualstate; /* Execution state for CHECK expr */ } NewConstraint; @@ -757,6 +760,25 @@ static List *GetParentedForeignKeyRefs(Relation partition); static void ATDetachCheckNoForeignKeyRefs(Relation partition); static char GetAttributeCompression(Oid atttypid, const char *compression); static char GetAttributeStorage(Oid atttypid, const char *storagemode); +static bool index_check_notnull(Relation relation, List *notnull_attnums); + +/* + * Ask phase 3 to verify that column attnum contains no NULLs. + * + * All requests for NOT NULL verification must come through here: + * ATRewriteTable may skip the table scan after checking, via an index, + * only the columns queued in tab->constraints. + */ +static void +queue_notnull_validation(AlteredTableInfo *tab, AttrNumber attnum) +{ + NewConstraint *newcon = palloc0_object(NewConstraint); + + newcon->contype = CONSTR_NOTNULL; + newcon->attnum = attnum; + tab->constraints = lappend(tab->constraints, newcon); + tab->verify_new_notnull = true; +} /* ---------------------------------------------------------------- @@ -6252,6 +6274,9 @@ ATRewriteTable(AlteredTableInfo *tab, Oid OIDNewHeap) needscan = true; con->qualstate = ExecPrepareExpr((Expr *) expand_generated_columns_in_expr(con->qual, oldrel, 1), estate); break; + case CONSTR_NOTNULL: + /* Nothing to do here */ + break; case CONSTR_FOREIGN: /* Nothing to do here */ break; @@ -6279,6 +6304,77 @@ ATRewriteTable(AlteredTableInfo *tab, Oid OIDNewHeap) notnull_attrs = notnull_virtual_attrs = NIL; if (newrel || tab->verify_new_notnull) { + bool verified_by_index = false; + + /* + * The conditions using indexscan mechanism fast verifying not-null + * constraints are quite strict. All of the following conditions must + * be met. + * + * 1. AlteredTableInfo->verify_new_notnull is true. + * + * 2. No table scan (e.g., for CHECK constraint verification) or table + * rewrite is expected later, if one is, using indexscan would wastes + * cycles. + * + * 3. Indexes cannot be created on virtual generated columns, so fast + * checking not-null constraints is not applicable to them. + * + * 4. The relation must be a plain table. + * + * 5. under AccessExclusiveLock no other transaction is in progress, + * so the index scan sees exactly the rows the table scan would. + */ + if (!needscan && + newrel == NULL && + oldrel->rd_rel->relkind == RELKIND_RELATION && + CheckRelationLockedByMe(oldrel, AccessExclusiveLock, false)) + { + List *notnull_attnums = NIL; + + Assert(!tab->rewrite); + + foreach(l, tab->constraints) + { + Form_pg_attribute attr; + NewConstraint *con = lfirst(l); + + if (con->contype != CONSTR_NOTNULL) + continue; + + attr = TupleDescAttr(newTupDesc, con->attnum - 1); + + if (attr->attisdropped) + continue; + + Assert(attr->attnotnull); + Assert(attr->attnum == con->attnum); + + if (attr->attgenerated == ATTRIBUTE_GENERATED_VIRTUAL) + { + needscan = true; + break; + } + + notnull_attnums = list_append_unique_int(notnull_attnums, + attr->attnum); + } + + if (!needscan && notnull_attnums != NIL) + { + if (!index_check_notnull(oldrel, notnull_attnums)) + needscan = true; + else + { + verified_by_index = true; + + ereport(DEBUG1, + errmsg_internal("all new not-null constraints on relation \"%s\" have been validated by using index scan", + RelationGetRelationName(oldrel))); + } + } + } + /* * If we are rebuilding the tuples OR if we added any new but not * verified not-null constraints, check all *valid* not-null @@ -6289,7 +6385,7 @@ ATRewriteTable(AlteredTableInfo *tab, Oid OIDNewHeap) * not-null constraints over virtual generated columns; instead, they * are collected in notnull_virtual_attrs for verification elsewhere. */ - for (i = 0; i < newTupDesc->natts; i++) + for (i = 0; !verified_by_index && i < newTupDesc->natts; i++) { CompactAttribute *attr = TupleDescCompactAttr(newTupDesc, i); @@ -6307,6 +6403,12 @@ ATRewriteTable(AlteredTableInfo *tab, Oid OIDNewHeap) } if (notnull_attrs || notnull_virtual_attrs) needscan = true; + + if (verified_by_index) + { + Assert(needscan == false); + Assert(newrel == NULL); + } } if (newrel || needscan) @@ -7648,13 +7750,13 @@ ATExecAddColumn(List **wqueue, AlteredTableInfo *tab, Relation rel, } } - if (!has_missing) + if (!has_missing && colDef->is_not_null) { /* * If the new column is NOT NULL, and there is no missing value, * tell Phase 3 it needs to check for NULLs. */ - tab->verify_new_notnull |= colDef->is_not_null; + queue_notnull_validation(tab, attribute->attnum); } } @@ -7960,7 +8062,8 @@ set_attnotnull(List **wqueue, Relation rel, AttrNumber attnum, AlteredTableInfo *tab; tab = ATGetQueueEntry(wqueue, rel); - tab->verify_new_notnull = true; + + queue_notnull_validation(tab, attnum); } CommandCounterIncrement(); @@ -8719,7 +8822,7 @@ ATExecSetExpression(AlteredTableInfo *tab, Relation rel, const char *colName, colName, RelationGetRelationName(rel)))); if (attgenerated == ATTRIBUTE_GENERATED_VIRTUAL && attTup->attnotnull) - tab->verify_new_notnull = true; + queue_notnull_validation(tab, attnum); /* * We need to prevent this because a change of expression could affect a @@ -10030,13 +10133,15 @@ ATAddCheckNNConstraint(List **wqueue, AlteredTableInfo *tab, Relation rel, { CookedConstraint *ccon = (CookedConstraint *) lfirst(lcon); - if (!ccon->skip_validation && ccon->contype != CONSTR_NOTNULL) + if (!ccon->skip_validation) { NewConstraint *newcon; newcon = palloc0_object(NewConstraint); newcon->name = ccon->name; newcon->contype = ccon->contype; + if (ccon->contype == CONSTR_NOTNULL) + newcon->attnum = ccon->attnum; newcon->qual = ccon->expr; tab->constraints = lappend(tab->constraints, newcon); @@ -13769,7 +13874,14 @@ QueueNNConstraintValidation(List **wqueue, Relation conrel, Relation rel, set_attnotnull(NULL, rel, attnum, false); tab = ATGetQueueEntry(wqueue, rel); - tab->verify_new_notnull = true; + + /* + * Queue validation for phase 3. Like ALTER TABLE SET NOT NULL, also add + * a NewConstraint to AlteredTableInfo->constraints: when ATRewriteTable + * verifies new not-null constraints with an index scan, it checks the + * columns queued there. + */ + queue_notnull_validation(tab, attnum); /* * Invalidate relcache so that others see the new validated constraint. @@ -22822,3 +22934,148 @@ GetAttributeStorage(Oid atttypid, const char *storagemode) return cstorage; } + + /* + * Try to verify newly added NOT NULL constraints using index scans. + * + * notnull_attnums: lists the columns of the newly added NOT NULL + * constraints. Each must be the leading column of a usable btree index, + * which we then scan for NULL entries. + * + * Returns true means the new NOT NULL constraints are fully verified and no + * extra table scans are necessary. + */ +static bool +index_check_notnull(Relation relation, List *notnull_attnums) +{ + List *idxs = NIL; + List *attnums = NIL; + bool found_null = false; + ListCell *lc, + *lc2; + Snapshot snapshot; + List *indexes; + Relation indexRel; + + Assert(notnull_attnums != NIL); + + indexes = RelationGetIndexList(relation); + foreach_oid(indexoid, indexes) + { + AttrNumber attnum; + Form_pg_index indexStruct; + + indexRel = index_open(indexoid, AccessShareLock); + indexStruct = indexRel->rd_index; + + /* + * We only use non-deferrable, valid, and live b-tree indexes to + * verify NOT NULL constraints. + */ + if (!indexStruct->indimmediate || + !indexStruct->indisvalid || + !indexStruct->indislive || + indexRel->rd_rel->relam != BTREE_AM_OID) + { + index_close(indexRel, NoLock); + continue; + } + + /* cannot use expression index or partial index too */ + if (!heap_attisnull(indexRel->rd_indextuple, Anum_pg_index_indexprs, NULL) || + !heap_attisnull(indexRel->rd_indextuple, Anum_pg_index_indpred, NULL)) + { + index_close(indexRel, NoLock); + continue; + } + + /* + * If the index is valid but cannot be used yet, ignore it. See + * src/backend/access/heap/README.HOT for discussion. + */ + if (indexStruct->indcheckxmin && + !TransactionIdPrecedes(HeapTupleHeaderGetXmin(indexRel->rd_indextuple->t_data), + TransactionXmin)) + { + index_close(indexRel, NoLock); + continue; + } + + attnum = (indexStruct->indkey.values[0]); + + if (list_member_int(notnull_attnums, attnum) && + !list_member_int(attnums, attnum)) + { + attnums = lappend_int(attnums, attnum); + idxs = lappend_oid(idxs, indexoid); + } + index_close(indexRel, NoLock); + } + + /* + * Verify NOT NULL constraints using a suitable index, falling back to a + * full table scan if a suitable index is not present. + */ + if (list_length(notnull_attnums) != list_length(attnums)) + return false; + + foreach_int(attno, notnull_attnums) + { + if (!list_member_int(attnums, attno)) + return false; + } + + snapshot = RegisterSnapshot(GetLatestSnapshot()); + + forboth(lc, attnums, lc2, idxs) + { + IndexScanDesc indexScan; + ScanKeyData scankeys[1]; + AttrNumber attno = lfirst_int(lc); + Oid indexoid = lfirst_oid(lc2); + TupleTableSlot *existing_slot; + + indexRel = index_open(indexoid, NoLock); + + Assert(indexRel->rd_index->indkey.values[0] == attno); + + existing_slot = table_slot_create(relation, NULL); + + /* set up an IS NULL scan key so that only NULLs are returned */ + ScanKeyEntryInitialize(&scankeys[0], + SK_ISNULL | SK_SEARCHNULL, + 1, /* index col to scan */ + InvalidStrategy, /* no strategy */ + InvalidOid, /* no strategy subtype */ + InvalidOid, /* no collation */ + InvalidOid, /* no reg proc for this */ + (Datum) 0); /* constant */ + + indexScan = index_beginscan(relation, + indexRel, + false, + snapshot, + NULL, + 1, + 0, + SO_NONE); + index_rescan(indexScan, scankeys, 1, NULL, 0); + + /* + * A returned tuple is a visible row with NULL in this column; btree + * never requests a recheck. Returning false makes the caller fall + * back to a full table scan. + */ + found_null = table_index_getnext_slot(indexScan, ForwardScanDirection, + existing_slot); + index_endscan(indexScan); + index_close(indexRel, NoLock); + ExecDropSingleTupleTableSlot(existing_slot); + + if (found_null) + break; + } + UnregisterSnapshot(snapshot); + + return !found_null; +} diff --git a/src/test/modules/test_misc/meson.build b/src/test/modules/test_misc/meson.build index 5d81f5b13be..ae62367d16e 100644 --- a/src/test/modules/test_misc/meson.build +++ b/src/test/modules/test_misc/meson.build @@ -24,6 +24,7 @@ tests += { 't/013_temp_obj_multisession.pl', 't/014_log_statement_max_length.pl', 't/015_temp_schema_exit_deferrable.pl', + 't/016_indexscan_validate_notnull.pl', ], # The injection points are cluster-wide, so disable installcheck 'runningcheck': false, diff --git a/src/test/modules/test_misc/t/016_indexscan_validate_notnull.pl b/src/test/modules/test_misc/t/016_indexscan_validate_notnull.pl new file mode 100644 index 00000000000..5e14ae622d7 --- /dev/null +++ b/src/test/modules/test_misc/t/016_indexscan_validate_notnull.pl @@ -0,0 +1,145 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# Verify that indexscan mechanism can speedup ALTER TABLE ADD NOT NULL + +use strict; +use warnings FATAL => 'all'; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; +use Time::HiRes qw(usleep); + +# Initialize a test cluster +my $node = PostgreSQL::Test::Cluster->new('primary'); +$node->init(); +# Turn message level up to DEBUG1 so that we get the messages we want to see +$node->append_conf('postgresql.conf', 'client_min_messages = DEBUG1'); +$node->start; + +# Run a SQL command and return psql's stderr (including debug messages) +sub run_sql_command +{ + my $sql = shift; + my $stderr; + + $node->psql( + 'postgres', + $sql, + stderr => \$stderr, + on_error_die => 1, + on_error_stop => 1); + return $stderr; +} + +# Verify that run_sql_command output confirms an index scan is used for the NOT NULL check. +sub is_indexscan_veritify_notnull +{ + my $output = shift; + return index($output, 'DEBUG: all new not-null constraints on relation') != -1; +} + +my $output; + +note "test alter table SET NOT NULL using indexscan with partitioned table"; + +run_sql_command( + 'CREATE TABLE tp_notnull ( + a int, b int, c int, d int, f1 int default 1, + f2 int default 2, f3 int default 3, f4 int default 4, f5 int default 5) PARTITION BY range(a); + CREATE TABLE tp_notnull_1 partition of tp_notnull for values from ( 1 ) to (10); + CREATE TABLE tp_notnull_2 partition of tp_notnull for values from ( 10 ) to (21); + INSERT INTO tp_notnull(a, b, c, d) SELECT g, g + 10, g + 15, g FROM generate_series(1,19) g; + CREATE INDEX ON tp_notnull(b); + CREATE INDEX ON tp_notnull(c); + CREATE INDEX ON tp_notnull(d); + CREATE INDEX ON tp_notnull(f1) INCLUDE (f2); + CREATE INDEX ON tp_notnull USING hash (f3 int4_ops); + '); + +# Using index scan for NOT NULL verification requires every not-null column must possess a suitable index +$output = run_sql_command( + 'ALTER TABLE tp_notnull ALTER COLUMN a SET NOT NULL, ADD CONSTRAINT c_nn NOT NULL c'); +ok(!is_indexscan_veritify_notnull($output), 'cannot use indexscan to verify not-null constraints on table tp_notnull'); + +# All columns have index on it, OK +$output = run_sql_command( + 'ALTER TABLE tp_notnull ALTER COLUMN d SET NOT NULL, ALTER COLUMN c SET NOT NULL, ALTER COLUMN b SET NOT NULL;'); +ok(is_indexscan_veritify_notnull($output), + 'using indexscan to verify not-null constraints on column b, c, d'); +ok( $output =~ + m/all new not-null constraints on relation "tp_notnull_1" have been validated by using index scan/, + 'all newly added constraints proved by indexscan'); +ok( $output =~ + m/all new not-null constraints on relation "tp_notnull_2" have been validated by using index scan/, + 'all newly added constraints proved by indexscan'); + +run_sql_command('ALTER TABLE tp_notnull ALTER COLUMN b DROP NOT NULL;'); +# Now using index scans on partitions to quickly verify NOT NULL constraints +$output = run_sql_command('ALTER TABLE tp_notnull_1 ALTER COLUMN b SET NOT NULL;'); +ok(is_indexscan_veritify_notnull($output), 'index scan verifies NOT NULL for column b'); + +run_sql_command('ALTER TABLE tp_notnull ALTER COLUMN b DROP NOT NULL;'); +# Cannot use indexscan to verify not-null constraint if table rewrite happens +$output = run_sql_command( + 'ALTER TABLE tp_notnull ALTER COLUMN b SET NOT NULL, ALTER COLUMN b SET DATA TYPE bigint;' +); +ok(!is_indexscan_veritify_notnull($output), 'table rewrite on tp_notnull, cannot use indexscan to verify not-null constraints on it'); + +run_sql_command('ALTER TABLE tp_notnull ALTER COLUMN d DROP NOT NULL'); + +# Cannot use indexscan because ALTER TABLE VALIDATE CONSTRAINT only hold +# ShareUpdateExclusiveLock lock on the table +run_sql_command('ALTER TABLE tp_notnull ADD CONSTRAINT tp_notnull_d NOT NULL d NOT VALID;'); +$output = run_sql_command( + 'ALTER TABLE tp_notnull VALIDATE CONSTRAINT tp_notnull_d;' +); +ok(!is_indexscan_veritify_notnull($output), 'cannot use indexscan for ALTER TABLE VALIDATE CONSTRAINT command'); + +run_sql_command('ALTER TABLE tp_notnull ALTER COLUMN d DROP NOT NULL; + ALTER TABLE tp_notnull ADD CONSTRAINT tp_notnull_d NOT NULL d NOT VALID;'); + +# OK, ALTER COLUMN SET NOT NULL acquires AccessExclusiveLock +$output = run_sql_command( + 'ALTER TABLE tp_notnull VALIDATE CONSTRAINT tp_notnull_d, ALTER COLUMN b SET NOT NULL;' +); +ok(is_indexscan_veritify_notnull($output), 'using indexscan to verify not-null constraints on column b, c, d'); + +# Indexscan mechanism to verify NOT NULL constraints cannot apply to index +# INCLUDE column +$output = run_sql_command( + 'ALTER TABLE tp_notnull ADD CONSTRAINT nnf4 NOT NULL f2;' +); +ok(!is_indexscan_veritify_notnull($output), 'cannot use indexscan to validate not-null constraint on tp_notnull'); + +# Indexscan mechanism to verify NOT NULL constraints cannot apply to non-Btree index +$output = run_sql_command('ALTER TABLE tp_notnull ALTER COLUMN f3 SET NOT NULL;'); +ok(!is_indexscan_veritify_notnull($output), 'cannot use indexscan to validate not-null constraints on tp_notnull'); + +run_sql_command('CREATE INDEX ON tp_notnull USING btree (f4 int4_ops, f5 int4_ops)'); +# Indexscan mechanism to verify NOT NULL constraints only apply to leading column +$output = run_sql_command('ALTER TABLE tp_notnull ALTER COLUMN f4 SET NOT NULL'); +ok(is_indexscan_veritify_notnull($output), 'cannot use indexscan to validate not-null constraints on tp_notnull'); + +$output = run_sql_command('ALTER TABLE tp_notnull ALTER COLUMN f5 SET NOT NULL'); +ok(!is_indexscan_veritify_notnull($output), 'cannot use indexscan to validate not-null constraints on tp_notnull'); + +run_sql_command(' + create table t(id int, v int) with (autovacuum_enabled = false, fillfactor = 50); + insert into t values (1, NULL)' +); +run_sql_command(' + begin; + select count(*) from t; + update t set v = 5 where id = 1; + create index t_v_idx on t (v); + alter table t alter column v set not null; + commit;' +); +# t_v_idx has pg_index.indcheckxmin = true: its underlying tuple belongs to a +# HOT chain, so this index cannot be trusted to answer "is this column NULL" +ok(!is_indexscan_veritify_notnull($output), 'cannot use indexscan to validate not-null constraints on t'); + +$node->stop('fast'); + +done_testing(); \ No newline at end of file -- 2.34.1