From a9b7e4a408448521719fb95f4171bc6a683f088b Mon Sep 17 00:00:00 2001 From: Bertrand Drouvot Date: Thu, 15 Oct 2026 11:53:29 +0000 Subject: [PATCH v1 3/4] Use bulk-write strategy for TOAST inserts Bulk insert operations use a BAS_BULKWRITE strategy for the main relation, but TOAST rows are inserted without it. Large amounts of TOAST data can therefore evict useful shared buffers. Give each BulkInsertState a state for its TOAST relation. Both states use the same strategy provided by the caller while keeping their relation local insertion state separate. Do not use bulk extension ranges for TOAST relations, as that could reserve too much unused space in small TOAST tables. Add tests for partitioned COPY and COPY's single-insert and multi-insert paths. Reported-by: Justin Pryzby Author: Bertrand Drouvot Reviewed-by: Discussion: https://postgr.es/m/ Discussion: https://postgr.es/m/20220622143841.GS22452%40telsasoft.com --- src/backend/access/common/toast_internals.c | 28 +++++++++-- src/backend/access/heap/heapam.c | 20 +++++--- src/backend/access/heap/heaptoast.c | 10 ++-- src/backend/access/heap/hio.c | 36 ++++++++----- src/backend/access/heap/rewriteheap.c | 2 +- src/backend/access/table/toast_helper.c | 5 +- src/include/access/heaptoast.h | 5 +- src/include/access/hio.h | 6 ++- src/include/access/tableam.h | 4 +- src/include/access/toast_helper.h | 5 +- src/include/access/toast_internals.h | 5 +- src/test/regress/expected/copy.out | 45 +++++++++++++++++ src/test/regress/expected/stats.out | 56 +++++++++++++++++++++ src/test/regress/sql/copy.sql | 53 +++++++++++++++++++ src/test/regress/sql/stats.sql | 50 ++++++++++++++++++ 15 files changed, 296 insertions(+), 34 deletions(-) 7.8% src/backend/access/common/ 22.7% src/backend/access/heap/ 9.3% src/include/access/ 31.0% src/test/regress/expected/ 27.9% src/test/regress/sql/ diff --git a/src/backend/access/common/toast_internals.c b/src/backend/access/common/toast_internals.c index 2bdc0b7d3f2..68508c03715 100644 --- a/src/backend/access/common/toast_internals.c +++ b/src/backend/access/common/toast_internals.c @@ -17,6 +17,7 @@ #include "access/genam.h" #include "access/heapam.h" #include "access/heaptoast.h" +#include "access/hio.h" #include "access/table.h" #include "access/toast_compression.h" #include "access/toast_internals.h" @@ -24,6 +25,7 @@ #include "catalog/catalog.h" #include "miscadmin.h" #include "utils/fmgroids.h" +#include "utils/memutils.h" #include "utils/rel.h" #include "utils/snapmgr.h" @@ -184,13 +186,15 @@ toast_preserve_valueid(Relation toastrel, Oid toastoid, * value: datum to be pushed to toast storage * oldexternal: if not NULL, toast pointer previously representing the datum * options: options to be passed to heap_insert() for toast rows + * bistate: bulk insert state for the main relation, or NULL * ---------- */ Datum -toast_save_datum(Relation rel, Datum value, - varlena *oldexternal, uint32 options) +toast_save_datum(Relation rel, Datum value, varlena *oldexternal, uint32 options, + BulkInsertState bistate) { Relation toastrel; + BulkInsertState toast_bistate = NULL; Relation *toastidxs; TupleDesc toasttupDesc; CommandId mycid = GetCurrentCommandId(true); @@ -220,6 +224,24 @@ toast_save_datum(Relation rel, Datum value, toastrel = table_open(rel->rd_rel->reltoastrelid, RowExclusiveLock); toasttupDesc = toastrel->rd_att; + if (bistate != NULL) + { + if (bistate->toast_state == NULL) + { + MemoryContext oldcontext; + + /* + * Allocate the TOAST state in the same context as its parent, as + * the current context may be reset before the bulk insert + * finishes. + */ + oldcontext = MemoryContextSwitchTo(GetMemoryChunkContext(bistate)); + bistate->toast_state = GetBulkInsertState(toastrel, bistate->strategy); + MemoryContextSwitchTo(oldcontext); + } + toast_bistate = bistate->toast_state; + } + /* Open all the toast indexes and look for the valid one */ validIndex = toast_open_indexes(toastrel, RowExclusiveLock, @@ -354,7 +376,7 @@ toast_save_datum(Relation rel, Datum value, toasttup = heap_form_tuple(toasttupDesc, t_values, t_isnull); - heap_insert(toastrel, toasttup, mycid, options, NULL); + heap_insert(toastrel, toasttup, mycid, options, toast_bistate); /* * Create the index entry. We cheat a little here by not using diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c index 6e526793c9d..3773e206541 100644 --- a/src/backend/access/heap/heapam.c +++ b/src/backend/access/heap/heapam.c @@ -58,7 +58,8 @@ static HeapTuple heap_prepare_insert(Relation relation, HeapTuple tup, - TransactionId xid, CommandId cid, uint32 options); + TransactionId xid, CommandId cid, uint32 options, + BulkInsertState bistate); static XLogRecPtr log_heap_update(Relation reln, Buffer oldbuf, Buffer vmbuffer_old, Buffer newbuf, Buffer vmbuffer_new, HeapTuple oldtup, @@ -1960,6 +1961,7 @@ GetBulkInsertState(Relation relation, BufferAccessStrategy strategy) bistate = (BulkInsertState) palloc_object(BulkInsertStateData); bistate->strategy = strategy; bistate->relation_id = RelationGetRelid(relation); + bistate->toast_state = NULL; bistate->current_buf = InvalidBuffer; bistate->next_free = InvalidBlockNumber; bistate->last_free = InvalidBlockNumber; @@ -1973,6 +1975,8 @@ GetBulkInsertState(Relation relation, BufferAccessStrategy strategy) void FreeBulkInsertState(BulkInsertState bistate) { + if (bistate->toast_state != NULL) + FreeBulkInsertState(bistate->toast_state); if (bistate->current_buf != InvalidBuffer) ReleaseBuffer(bistate->current_buf); pfree(bistate); @@ -1986,6 +1990,9 @@ FreeBulkInsertState(BulkInsertState bistate) void ReleaseBulkInsertStatePin(BulkInsertState bistate) { + if (bistate->toast_state != NULL) + ReleaseBulkInsertStatePin(bistate->toast_state); + if (bistate->current_buf != InvalidBuffer) ReleaseBuffer(bistate->current_buf); bistate->current_buf = InvalidBuffer; @@ -2039,7 +2046,7 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid, * Note: below this point, heaptup is the data we actually intend to store * into the relation; tup is the caller's original untoasted data. */ - heaptup = heap_prepare_insert(relation, tup, xid, cid, options); + heaptup = heap_prepare_insert(relation, tup, xid, cid, options, bistate); /* * Find buffer to insert this tuple into. If the page is all visible, @@ -2240,7 +2247,7 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid, */ static HeapTuple heap_prepare_insert(Relation relation, HeapTuple tup, TransactionId xid, - CommandId cid, uint32 options) + CommandId cid, uint32 options, BulkInsertState bistate) { /* * To allow parallel inserts, we need to ensure that they are safe to be @@ -2276,7 +2283,7 @@ heap_prepare_insert(Relation relation, HeapTuple tup, TransactionId xid, return tup; } else if (HeapTupleHasExternal(tup) || tup->t_len > TOAST_TUPLE_THRESHOLD) - return heap_toast_insert_or_update(relation, tup, NULL, options); + return heap_toast_insert_or_update(relation, tup, NULL, options, bistate); else return tup; } @@ -2358,7 +2365,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples, slots[i]->tts_tableOid = RelationGetRelid(relation); tuple->t_tableOid = slots[i]->tts_tableOid; heaptuples[i] = heap_prepare_insert(relation, tuple, xid, cid, - options); + options, bistate); } /* @@ -3993,7 +4000,8 @@ l2: * tuples too. We never skip the FSM here. */ heaptup = heap_toast_insert_or_update(relation, newtup, &oldtup, - walLogical ? 0 : HEAP_INSERT_NO_LOGICAL); + walLogical ? 0 : HEAP_INSERT_NO_LOGICAL, + NULL); newtupsize = MAXALIGN(heaptup->t_len); } else diff --git a/src/backend/access/heap/heaptoast.c b/src/backend/access/heap/heaptoast.c index b03d8492959..345afaf622c 100644 --- a/src/backend/access/heap/heaptoast.c +++ b/src/backend/access/heap/heaptoast.c @@ -84,6 +84,7 @@ heap_toast_delete(Relation rel, HeapTuple oldtup, bool is_speculative) * newtup: the candidate new tuple to be inserted * oldtup: the old row version for UPDATE, or NULL for INSERT * options: options to be passed to heap_insert() for toast rows + * bistate: bulk insert state for INSERT, or NULL for UPDATE * Result: * either newtup if no toasting is needed, or a palloc'd modified tuple * that is what should actually get stored @@ -94,7 +95,7 @@ heap_toast_delete(Relation rel, HeapTuple oldtup, bool is_speculative) */ HeapTuple heap_toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, - uint32 options) + uint32 options, BulkInsertState bistate) { HeapTuple result_tuple; TupleDesc tupleDesc; @@ -124,6 +125,7 @@ heap_toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, */ Assert(rel->rd_rel->relkind == RELKIND_RELATION || rel->rd_rel->relkind == RELKIND_MATVIEW); + Assert(bistate == NULL || oldtup == NULL); /* * Get the tuple descriptor and break down the tuple(s) into fields. @@ -214,7 +216,7 @@ heap_toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, */ if (toast_attr[biggest_attno].tai_size > maxDataLen && rel->rd_rel->reltoastrelid != InvalidOid) - toast_tuple_externalize(&ttc, biggest_attno, options); + toast_tuple_externalize(&ttc, biggest_attno, options, bistate); } /* @@ -231,7 +233,7 @@ heap_toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, biggest_attno = toast_tuple_find_biggest_attribute(&ttc, false, false); if (biggest_attno < 0) break; - toast_tuple_externalize(&ttc, biggest_attno, options); + toast_tuple_externalize(&ttc, biggest_attno, options, bistate); } /* @@ -267,7 +269,7 @@ heap_toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, if (biggest_attno < 0) break; - toast_tuple_externalize(&ttc, biggest_attno, options); + toast_tuple_externalize(&ttc, biggest_attno, options, bistate); } /* diff --git a/src/backend/access/heap/hio.c b/src/backend/access/heap/hio.c index fb4ffb97592..4cb7821277e 100644 --- a/src/backend/access/heap/hio.c +++ b/src/backend/access/heap/hio.c @@ -244,15 +244,23 @@ RelationAddBlocks(Relation relation, BulkInsertState bistate, uint32 not_in_fsm_pages; Buffer buffer; Page page; + bool use_extension_state; + + /* + * TOAST writes share the bulk ring and retain a target-page pin, but do + * not reserve private extension ranges that can leave large unused tails + * in small TOAST relations. + */ + use_extension_state = bistate != NULL && relation->rd_rel->relkind != RELKIND_TOASTVALUE; /* * Determine by how many pages to try to extend by. */ - if (bistate == NULL && !use_fsm) + if (!use_extension_state && !use_fsm) { /* - * If we have neither bistate, nor can use the FSM, we can't bulk - * extend - there'd be no way to find the additional pages. + * Without the FSM or relation extension state, there'd be no way to + * find additional pages after extending. */ extend_by_pages = 1; } @@ -295,7 +303,7 @@ RelationAddBlocks(Relation relation, BulkInsertState bistate, * prevent future contention. * --- */ - if (bistate) + if (use_extension_state) extend_by_pages = Max(extend_by_pages, bistate->already_extended_by); /* @@ -308,14 +316,14 @@ RelationAddBlocks(Relation relation, BulkInsertState bistate, /* * How many of the extended pages should be entered into the FSM? * - * If we have a bistate, only enter pages that we don't need ourselves - * into the FSM. Otherwise every other backend will immediately try to - * use the pages this backend needs for itself, causing unnecessary - * contention. If we don't have a bistate, we can't avoid the FSM. + * If we retain private extension state, only enter pages that we don't + * need ourselves into the FSM. Otherwise every other backend will + * immediately try to use the pages this backend needs for itself, causing + * unnecessary contention. * * Never enter the page returned into the FSM, we'll immediately use it. */ - if (num_pages > 1 && bistate == NULL) + if (num_pages > 1 && !use_extension_state) not_in_fsm_pages = 1; else not_in_fsm_pages = num_pages; @@ -404,7 +412,7 @@ RelationAddBlocks(Relation relation, BulkInsertState bistate, FreeSpaceMapVacuumRange(relation, first_fsm_block, last_block + 1); } - if (bistate) + if (use_extension_state) { /* * Remember the additional pages we extended by, so we later can use @@ -421,10 +429,14 @@ RelationAddBlocks(Relation relation, BulkInsertState bistate, bistate->last_free = InvalidBlockNumber; } - /* maintain bistate->current_buf */ + bistate->already_extended_by += extend_by_pages; + } + + /* maintain bistate->current_buf */ + if (bistate) + { IncrBufferRefCount(buffer); bistate->current_buf = buffer; - bistate->already_extended_by += extend_by_pages; } return buffer; diff --git a/src/backend/access/heap/rewriteheap.c b/src/backend/access/heap/rewriteheap.c index 0648e433031..90ddfd11908 100644 --- a/src/backend/access/heap/rewriteheap.c +++ b/src/backend/access/heap/rewriteheap.c @@ -628,7 +628,7 @@ raw_heap_insert(RewriteState state, HeapTuple tup) options |= HEAP_INSERT_NO_LOGICAL; heaptup = heap_toast_insert_or_update(state->rs_new_rel, tup, NULL, - options); + options, NULL); } else heaptup = tup; diff --git a/src/backend/access/table/toast_helper.c b/src/backend/access/table/toast_helper.c index 2613d9dc095..d550599ee13 100644 --- a/src/backend/access/table/toast_helper.c +++ b/src/backend/access/table/toast_helper.c @@ -269,7 +269,8 @@ toast_tuple_try_compression(ToastTupleContext *ttc, int attribute) * Move an attribute to external storage. */ void -toast_tuple_externalize(ToastTupleContext *ttc, int attribute, uint32 options) +toast_tuple_externalize(ToastTupleContext *ttc, int attribute, uint32 options, + struct BulkInsertStateData *bistate) { Datum *value = &ttc->ttc_values[attribute]; Datum old_value = *value; @@ -277,7 +278,7 @@ toast_tuple_externalize(ToastTupleContext *ttc, int attribute, uint32 options) attr->tai_colflags |= TOASTCOL_IGNORE; *value = toast_save_datum(ttc->ttc_rel, old_value, attr->tai_oldexternal, - options); + options, bistate); if ((attr->tai_colflags & TOASTCOL_NEEDS_FREE) != 0) pfree(DatumGetPointer(old_value)); attr->tai_colflags |= TOASTCOL_NEEDS_FREE; diff --git a/src/include/access/heaptoast.h b/src/include/access/heaptoast.h index 4c3fda8a5f9..471883821ac 100644 --- a/src/include/access/heaptoast.h +++ b/src/include/access/heaptoast.h @@ -17,6 +17,8 @@ #include "storage/lockdefs.h" #include "utils/relcache.h" +struct BulkInsertStateData; + /* * Find the maximum size of a tuple if there are to be N tuples per page. */ @@ -109,7 +111,8 @@ * ---------- */ extern HeapTuple heap_toast_insert_or_update(Relation rel, HeapTuple newtup, - HeapTuple oldtup, uint32 options); + HeapTuple oldtup, uint32 options, + struct BulkInsertStateData *bistate); /* ---------- * heap_toast_delete - diff --git a/src/include/access/hio.h b/src/include/access/hio.h index 5d02d5426e2..b93267cace1 100644 --- a/src/include/access/hio.h +++ b/src/include/access/hio.h @@ -19,10 +19,11 @@ #include "utils/relcache.h" /* - * state for bulk inserts into one relation --- private to heapam.c and hio.c + * state for bulk inserts into one relation * * The caller owns the buffer access strategy and may share it with states for - * other relations in the same bulk operation. If current_buf isn't + * related relations in the same bulk operation. The heap AM can create a child + * state for the the relation's TOAST table. If current_buf isn't * InvalidBuffer, then we are holding an extra pin on that buffer. * * "typedef struct BulkInsertStateData *BulkInsertState" is in heapam.h @@ -31,6 +32,7 @@ typedef struct BulkInsertStateData { BufferAccessStrategy strategy; /* our BULKWRITE strategy object */ Oid relation_id; /* relation this state belongs to */ + struct BulkInsertStateData *toast_state; /* state for TOAST relation */ Buffer current_buf; /* current insertion target page */ /* diff --git a/src/include/access/tableam.h b/src/include/access/tableam.h index 82d5a8717e2..a927138d492 100644 --- a/src/include/access/tableam.h +++ b/src/include/access/tableam.h @@ -1466,7 +1466,9 @@ table_index_delete_tuples(Relation rel, TM_IndexDeleteOp *delstate) * * A BulkInsertState object (if any; bistate can be NULL for default behavior) * belongs to one relation and allows the AM to optimize a series of inserts. - * If `bistate` is provided, table_finish_bulk_insert() needs to be called. + * The heap AM creates a related state for the TOAST relation that shares the + * same buffer access strategy. If `bistate` is provided, + * table_finish_bulk_insert() needs to be called. * * On return the slot's tts_tid and tts_tableOid are updated to reflect the * insertion. But note that any toasting of fields within the slot is NOT diff --git a/src/include/access/toast_helper.h b/src/include/access/toast_helper.h index 2ec92397f26..57a27755761 100644 --- a/src/include/access/toast_helper.h +++ b/src/include/access/toast_helper.h @@ -16,6 +16,8 @@ #include "utils/rel.h" +struct BulkInsertStateData; + /* * Information about one column of a tuple being toasted. * @@ -107,7 +109,8 @@ extern int toast_tuple_find_biggest_attribute(ToastTupleContext *ttc, bool check_main); extern void toast_tuple_try_compression(ToastTupleContext *ttc, int attribute); extern void toast_tuple_externalize(ToastTupleContext *ttc, int attribute, - uint32 options); + uint32 options, + struct BulkInsertStateData *bistate); extern void toast_tuple_cleanup(ToastTupleContext *ttc); extern void toast_delete_external(Relation rel, const Datum *values, const bool *isnull, diff --git a/src/include/access/toast_internals.h b/src/include/access/toast_internals.h index a31b47f8df5..4ac648642fb 100644 --- a/src/include/access/toast_internals.h +++ b/src/include/access/toast_internals.h @@ -17,12 +17,15 @@ #include "utils/relcache.h" #include "utils/snapshot.h" +struct BulkInsertStateData; + extern Datum toast_compress_datum(Datum value, char cmethod); extern Oid toast_get_valid_index(Oid toastoid, LOCKMODE lock); extern void toast_delete_datum(Relation rel, Datum value, bool is_speculative); extern Datum toast_save_datum(Relation rel, Datum value, - varlena *oldexternal, uint32 options); + varlena *oldexternal, uint32 options, + struct BulkInsertStateData *bistate); extern void toast_valueid_scankey_init(ScanKey entry, Oid toast_typid, Oid8 valueid); diff --git a/src/test/regress/expected/copy.out b/src/test/regress/expected/copy.out index 979bef5f3f2..2de2adc66a0 100644 --- a/src/test/regress/expected/copy.out +++ b/src/test/regress/expected/copy.out @@ -572,6 +572,51 @@ SELECT tableoid::regclass, id % 2 = 0 is_even, count(*) from parted_si GROUP BY (2 rows) DROP TABLE parted_si; +-- Test partition switching with toasted rows in COPY's single-insert path +CREATE TABLE parted_si_toast ( + id int not null, + payload text GENERATED ALWAYS AS (repeat(id::text, 10000)) STORED, + -- Force COPY's single-insert path with a volatile default + rand float8 not null default random() +) +PARTITION BY LIST((id % 2)); +-- Ensure that generated payloads are stored in the TOAST relations +ALTER TABLE parted_si_toast ALTER COLUMN payload SET STORAGE EXTERNAL; +CREATE TABLE parted_si_toast_even PARTITION OF parted_si_toast FOR VALUES IN (0); +CREATE TABLE parted_si_toast_odd PARTITION OF parted_si_toast FOR VALUES IN (1); +COPY parted_si_toast(id) FROM stdin; +SELECT tableoid::regclass, count(*), + bool_and(payload = repeat(id::text, 10000)) AS payload_ok + FROM parted_si_toast GROUP BY 1 ORDER BY 1; + tableoid | count | payload_ok +----------------------+-------+------------ + parted_si_toast_even | 2 | t + parted_si_toast_odd | 2 | t +(2 rows) + +DROP TABLE parted_si_toast; +-- Check that a TOAST bulk insertion state survives a per-tuple context reset +-- between multi-insert flushes. +CREATE TABLE parted_mi_toast ( + id int not null, + payload text GENERATED ALWAYS AS (repeat(id::text, 10000)) STORED +) +PARTITION BY LIST((id % 2)); +ALTER TABLE parted_mi_toast ALTER COLUMN payload SET STORAGE EXTERNAL; +CREATE TABLE parted_mi_toast_even PARTITION OF parted_mi_toast FOR VALUES IN (0); +CREATE TABLE parted_mi_toast_odd PARTITION OF parted_mi_toast FOR VALUES IN (1); +CREATE TRIGGER part_ins_trig + BEFORE INSERT ON parted_mi_toast_odd + FOR EACH ROW EXECUTE FUNCTION part_ins_func(); +COPY parted_mi_toast(id) FROM stdin; +SELECT count(*), bool_and(payload = repeat(id::text, 10000)) AS payload_ok + FROM parted_mi_toast; + count | payload_ok +-------+------------ + 3 | t +(1 row) + +DROP TABLE parted_mi_toast; -- ensure COPY FREEZE errors for foreign tables begin; create foreign data wrapper copytest_wrapper; diff --git a/src/test/regress/expected/stats.out b/src/test/regress/expected/stats.out index 8b15471248b..1f041617aee 100644 --- a/src/test/regress/expected/stats.out +++ b/src/test/regress/expected/stats.out @@ -1900,6 +1900,62 @@ SELECT :io_sum_bulkwrite_strategy_extends_after > :io_sum_bulkwrite_strategy_ext t (1 row) +-- Test that TOAST relations use the BAS_BULKWRITE strategy during single and +-- multi insert COPY. +CREATE TABLE test_io_bulkwrite_toast ( + id int, + payload text NOT NULL DEFAULT repeat('x', 10000), + force_single_insert float8 DEFAULT random() +); +ALTER TABLE test_io_bulkwrite_toast ALTER COLUMN payload SET STORAGE EXTERNAL; +-- Make COPY reuse the main relation's first page, so that relation extensions +-- measured below belong to the TOAST relation. +INSERT INTO test_io_bulkwrite_toast (id, payload) VALUES (-1, 'x'); +-- The volatile default makes COPY use the single insert path +SELECT sum(extends) AS io_sum_bulkwrite_toast_single_extends_before + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE context = 'bulkwrite' \gset +COPY test_io_bulkwrite_toast(id) FROM stdin; +SELECT pg_stat_force_next_flush(); + pg_stat_force_next_flush +-------------------------- + +(1 row) + +SELECT sum(extends) AS io_sum_bulkwrite_toast_single_extends_after + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE context = 'bulkwrite' \gset +SELECT :io_sum_bulkwrite_toast_single_extends_after > + :io_sum_bulkwrite_toast_single_extends_before; + ?column? +---------- + t +(1 row) + +-- Dropping the volatile default makes COPY use the multi insert path +ALTER TABLE test_io_bulkwrite_toast + ALTER COLUMN force_single_insert DROP DEFAULT; +SELECT sum(extends) AS io_sum_bulkwrite_toast_multi_extends_before + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE context = 'bulkwrite' \gset +COPY test_io_bulkwrite_toast(id) FROM stdin; +SELECT pg_stat_force_next_flush(); + pg_stat_force_next_flush +-------------------------- + +(1 row) + +SELECT sum(extends) AS io_sum_bulkwrite_toast_multi_extends_after + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE context = 'bulkwrite' \gset +SELECT :io_sum_bulkwrite_toast_multi_extends_after > + :io_sum_bulkwrite_toast_multi_extends_before; + ?column? +---------- + t +(1 row) + +DROP TABLE test_io_bulkwrite_toast; -- Test IO stats reset SELECT pg_stat_have_stats('io', 0, 0); pg_stat_have_stats diff --git a/src/test/regress/sql/copy.sql b/src/test/regress/sql/copy.sql index 6d3923fb3f0..d63ba5f9ced 100644 --- a/src/test/regress/sql/copy.sql +++ b/src/test/regress/sql/copy.sql @@ -524,6 +524,59 @@ SELECT tableoid::regclass, id % 2 = 0 is_even, count(*) from parted_si GROUP BY DROP TABLE parted_si; +-- Test partition switching with toasted rows in COPY's single-insert path +CREATE TABLE parted_si_toast ( + id int not null, + payload text GENERATED ALWAYS AS (repeat(id::text, 10000)) STORED, + -- Force COPY's single-insert path with a volatile default + rand float8 not null default random() +) +PARTITION BY LIST((id % 2)); + +-- Ensure that generated payloads are stored in the TOAST relations +ALTER TABLE parted_si_toast ALTER COLUMN payload SET STORAGE EXTERNAL; +CREATE TABLE parted_si_toast_even PARTITION OF parted_si_toast FOR VALUES IN (0); +CREATE TABLE parted_si_toast_odd PARTITION OF parted_si_toast FOR VALUES IN (1); + +COPY parted_si_toast(id) FROM stdin; +0 +1 +2 +3 +\. + +SELECT tableoid::regclass, count(*), + bool_and(payload = repeat(id::text, 10000)) AS payload_ok + FROM parted_si_toast GROUP BY 1 ORDER BY 1; + +DROP TABLE parted_si_toast; + +-- Check that a TOAST bulk insertion state survives a per-tuple context reset +-- between multi-insert flushes. +CREATE TABLE parted_mi_toast ( + id int not null, + payload text GENERATED ALWAYS AS (repeat(id::text, 10000)) STORED +) +PARTITION BY LIST((id % 2)); + +ALTER TABLE parted_mi_toast ALTER COLUMN payload SET STORAGE EXTERNAL; +CREATE TABLE parted_mi_toast_even PARTITION OF parted_mi_toast FOR VALUES IN (0); +CREATE TABLE parted_mi_toast_odd PARTITION OF parted_mi_toast FOR VALUES IN (1); +CREATE TRIGGER part_ins_trig + BEFORE INSERT ON parted_mi_toast_odd + FOR EACH ROW EXECUTE FUNCTION part_ins_func(); + +COPY parted_mi_toast(id) FROM stdin; +0 +1 +2 +\. + +SELECT count(*), bool_and(payload = repeat(id::text, 10000)) AS payload_ok + FROM parted_mi_toast; + +DROP TABLE parted_mi_toast; + -- ensure COPY FREEZE errors for foreign tables begin; create foreign data wrapper copytest_wrapper; diff --git a/src/test/regress/sql/stats.sql b/src/test/regress/sql/stats.sql index 674637e172b..7c11c07b4ab 100644 --- a/src/test/regress/sql/stats.sql +++ b/src/test/regress/sql/stats.sql @@ -902,6 +902,56 @@ SELECT sum(extends) AS io_sum_bulkwrite_strategy_extends_after FROM pg_stat_io WHERE context = 'bulkwrite' \gset SELECT :io_sum_bulkwrite_strategy_extends_after > :io_sum_bulkwrite_strategy_extends_before; +-- Test that TOAST relations use the BAS_BULKWRITE strategy during single and +-- multi insert COPY. +CREATE TABLE test_io_bulkwrite_toast ( + id int, + payload text NOT NULL DEFAULT repeat('x', 10000), + force_single_insert float8 DEFAULT random() +); +ALTER TABLE test_io_bulkwrite_toast ALTER COLUMN payload SET STORAGE EXTERNAL; + +-- Make COPY reuse the main relation's first page, so that relation extensions +-- measured below belong to the TOAST relation. +INSERT INTO test_io_bulkwrite_toast (id, payload) VALUES (-1, 'x'); + +-- The volatile default makes COPY use the single insert path +SELECT sum(extends) AS io_sum_bulkwrite_toast_single_extends_before + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE context = 'bulkwrite' \gset +COPY test_io_bulkwrite_toast(id) FROM stdin; +0 +1 +2 +3 +\. +SELECT pg_stat_force_next_flush(); +SELECT sum(extends) AS io_sum_bulkwrite_toast_single_extends_after + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE context = 'bulkwrite' \gset +SELECT :io_sum_bulkwrite_toast_single_extends_after > + :io_sum_bulkwrite_toast_single_extends_before; + +-- Dropping the volatile default makes COPY use the multi insert path +ALTER TABLE test_io_bulkwrite_toast + ALTER COLUMN force_single_insert DROP DEFAULT; +SELECT sum(extends) AS io_sum_bulkwrite_toast_multi_extends_before + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE context = 'bulkwrite' \gset +COPY test_io_bulkwrite_toast(id) FROM stdin; +4 +5 +6 +7 +\. +SELECT pg_stat_force_next_flush(); +SELECT sum(extends) AS io_sum_bulkwrite_toast_multi_extends_after + FROM pg_stat_get_backend_io(pg_backend_pid()) + WHERE context = 'bulkwrite' \gset +SELECT :io_sum_bulkwrite_toast_multi_extends_after > + :io_sum_bulkwrite_toast_multi_extends_before; +DROP TABLE test_io_bulkwrite_toast; + -- Test IO stats reset SELECT pg_stat_have_stats('io', 0, 0); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_pre_reset -- 2.34.1