From 647edfdc6984a9d4f53a2321f138e83c0a36df93 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Mon, 14 Sep 2026 13:08:51 +0900 Subject: [PATCH v18 1/5] Add support for oid8 TOAST values This commit adds the possibility to define TOAST tables with oid8 as value ID, based on the reloption toast_value_type. All the external TOAST pointers still rely on varatt_external and a single vartag. The values inserted in the oid8 TOAST tables are fed from the OID8 value generator, unfortunately casted to OID for now, as we do not have a vartag_external able to store Oid8 values yet. This will be changed in an upcoming patch that adds more vartag_external types and its associated structures, with the code being able to use a different external TOAST pointer depending on the attribute type of chunk_id in TOAST relations. All the changes done here are mechanical, with all the TOAST code able to do chunk ID lookups based on the two types now supported. Note that as of this commit, TOAST table can use oid8 for their chunk_id attribute, however the data is still stored on the HEAD side using 4-byte wide Oids. A follow-up change that introduces a new vartag_external will stick 8-byte Oids later. XXX: Catalog version bump required. --- src/include/access/toast_internals.h | 4 + src/include/catalog/pg_opclass.dat | 3 +- src/include/utils/rel.h | 1 + src/backend/access/common/reloptions.c | 3 +- src/backend/access/common/toast_internals.c | 89 +++++++++++++------ src/backend/access/heap/heaptoast.c | 7 +- src/backend/catalog/toasting.c | 25 +++++- .../replication/logical/reorderbuffer.c | 9 +- src/test/regress/expected/reloptions.out | 15 +++- src/test/regress/sql/reloptions.sql | 8 ++ doc/src/sgml/ref/create_table.sgml | 5 +- doc/src/sgml/storage.sgml | 7 +- contrib/amcheck/verify_heapam.c | 13 +-- 13 files changed, 143 insertions(+), 46 deletions(-) diff --git a/src/include/access/toast_internals.h b/src/include/access/toast_internals.h index bf45889a6428..e03cc1204beb 100644 --- a/src/include/access/toast_internals.h +++ b/src/include/access/toast_internals.h @@ -12,6 +12,7 @@ #ifndef TOAST_INTERNALS_H #define TOAST_INTERNALS_H +#include "access/skey.h" #include "access/toast_compression.h" #include "storage/lockdefs.h" #include "utils/relcache.h" @@ -52,6 +53,9 @@ extern void toast_delete_datum(Relation rel, Datum value, bool is_speculative); extern Datum toast_save_datum(Relation rel, Datum value, varlena *oldexternal, uint32 options); +extern void toast_valueid_scankey_init(ScanKey entry, Oid toast_typid, + Oid8 valueid); + extern int toast_open_indexes(Relation toastrel, LOCKMODE lock, Relation **toastidxs, diff --git a/src/include/catalog/pg_opclass.dat b/src/include/catalog/pg_opclass.dat index df170b80840b..b84c2bb7a8c3 100644 --- a/src/include/catalog/pg_opclass.dat +++ b/src/include/catalog/pg_opclass.dat @@ -179,7 +179,8 @@ opcintype => 'xid8' }, { opcmethod => 'hash', opcname => 'oid8_ops', opcfamily => 'hash/oid8_ops', opcintype => 'oid8' }, -{ opcmethod => 'btree', opcname => 'oid8_ops', opcfamily => 'btree/oid8_ops', +{ oid => '8285', oid_symbol => 'OID8_BTREE_OPS_OID', + opcmethod => 'btree', opcname => 'oid8_ops', opcfamily => 'btree/oid8_ops', opcintype => 'oid8' }, { opcmethod => 'hash', opcname => 'cid_ops', opcfamily => 'hash/cid_ops', opcintype => 'cid' }, diff --git a/src/include/utils/rel.h b/src/include/utils/rel.h index a88549dcc555..d61432de705e 100644 --- a/src/include/utils/rel.h +++ b/src/include/utils/rel.h @@ -346,6 +346,7 @@ typedef enum StdRdOptToastValueType { STDRD_OPTION_TOAST_VALUE_TYPE_INVALID = 0, STDRD_OPTION_TOAST_VALUE_TYPE_OID, + STDRD_OPTION_TOAST_VALUE_TYPE_OID8, } StdRdOptToastValueType; typedef struct StdRdOptions diff --git a/src/backend/access/common/reloptions.c b/src/backend/access/common/reloptions.c index 82491a31a9a4..ea9a04179090 100644 --- a/src/backend/access/common/reloptions.c +++ b/src/backend/access/common/reloptions.c @@ -553,6 +553,7 @@ static relopt_enum_elt_def StdRdOptToastValueTypes[] = { /* no value for INVALID */ {"oid", STDRD_OPTION_TOAST_VALUE_TYPE_OID}, + {"oid8", STDRD_OPTION_TOAST_VALUE_TYPE_OID8}, {(const char *) NULL} /* list terminator */ }; @@ -578,7 +579,7 @@ static relopt_enum enumRelOpts[] = }, StdRdOptToastValueTypes, STDRD_OPTION_TOAST_VALUE_TYPE_OID, - gettext_noop("Valid values are \"oid\".") + gettext_noop("Valid values are \"oid\" and \"oid8\".") }, { { diff --git a/src/backend/access/common/toast_internals.c b/src/backend/access/common/toast_internals.c index ca5ed8dfabca..de4b17fa8164 100644 --- a/src/backend/access/common/toast_internals.c +++ b/src/backend/access/common/toast_internals.c @@ -25,6 +25,7 @@ #include "utils/fmgroids.h" #include "utils/rel.h" #include "utils/snapmgr.h" +#include "utils/lsyscache.h" static bool toastrel_valueid_exists(Relation toastrel, Oid8 valueid); static bool toastid_valueid_exists(Oid toastrelid, Oid8 valueid); @@ -131,6 +132,7 @@ toast_save_datum(Relation rel, Datum value, Pointer dval = DatumGetPointer(value); int num_indexes; int validIndex; + Oid toast_typid = get_atttype(rel->rd_rel->reltoastrelid, 1); Assert(!VARATT_IS_EXTERNAL(dval)); @@ -200,24 +202,32 @@ toast_save_datum(Relation rel, Datum value, toast_pointer.va_toastrelid = RelationGetRelid(toastrel); /* - * Choose an OID to use as the value ID for this toast value. + * Choose a new value to use as the value ID for this toast value, be it + * for OID or OID8 TOAST relations. * - * Normally we just choose an unused OID within the toast table. But + * Normally we just choose an unused value within the toast table. But * during table-rewriting operations where we are preserving an existing - * toast table OID, we want to preserve toast value OIDs too. So, if + * toast table OID, we want to preserve toast value IDs too. So, if * rd_toastoid is set and we had a prior external value from that same * toast table, re-use its value ID. If we didn't have a prior external * value (which is a corner case, but possible if the table's attstorage * options have been changed), we have to pick a value ID that doesn't - * conflict with either new or existing toast value OIDs. + * conflict with either new or existing toast value IDs. If the TOAST + * table uses 8-byte value IDs, we should not really care much about + * that. */ if (!OidIsValid(rel->rd_toastoid)) { /* normal case: just choose an unused OID */ - toast_pointer.va_valueid = - GetNewOidWithIndex(toastrel, - RelationGetRelid(toastidxs[validIndex]), - (AttrNumber) 1); + if (toast_typid == OID8OID) + toast_pointer.va_valueid = GetNewObjectId8(); + else + { + toast_pointer.va_valueid = + GetNewOidWithIndex(toastrel, + RelationGetRelid(toastidxs[validIndex]), + (AttrNumber) 1); + } } else { @@ -263,17 +273,22 @@ toast_save_datum(Relation rel, Datum value, if (toast_pointer.va_valueid == InvalidOid) { /* - * new value; must choose an OID that doesn't conflict in either - * old or new toast table + * new value; must choose a value that doesn't conflict in either + * old or new toast table. */ - do + if (toast_typid == OID8OID) + toast_pointer.va_valueid = GetNewObjectId8(); + else { - toast_pointer.va_valueid = - GetNewOidWithIndex(toastrel, - RelationGetRelid(toastidxs[validIndex]), - (AttrNumber) 1); - } while (toastid_valueid_exists(rel->rd_toastoid, - toast_pointer.va_valueid)); + do + { + toast_pointer.va_valueid = + GetNewOidWithIndex(toastrel, + RelationGetRelid(toastidxs[validIndex]), + (AttrNumber) 1); + } while (toastid_valueid_exists(rel->rd_toastoid, + toast_pointer.va_valueid)); + } } } @@ -303,7 +318,10 @@ toast_save_datum(Relation rel, Datum value, /* * Build a tuple and store it */ - t_values[0] = ObjectIdGetDatum(toast_pointer.va_valueid); + if (toast_typid == OID8OID) + t_values[0] = ObjectId8GetDatum(toast_pointer.va_valueid); + else + t_values[0] = ObjectIdGetDatum(toast_pointer.va_valueid); t_values[1] = Int32GetDatum(chunk_seq++); SET_VARSIZE(&chunk_data, chunk_size + VARHDRSZ); memcpy(VARDATA(&chunk_data), data_p, chunk_size); @@ -366,6 +384,26 @@ toast_save_datum(Relation rel, Datum value, return PointerGetDatum(result); } +/* ---------- + * toast_valueid_scankey_init - + * + * Initialize a scan key that matches the value ID column of a TOAST table. + * ---------- + */ +void +toast_valueid_scankey_init(ScanKey entry, Oid toast_typid, Oid8 valueid) +{ + Assert(toast_typid == OIDOID || toast_typid == OID8OID); + if (toast_typid == OID8OID) + ScanKeyInit(entry, (AttrNumber) 1, + BTEqualStrategyNumber, F_OID8EQ, + ObjectId8GetDatum(valueid)); + else + ScanKeyInit(entry, (AttrNumber) 1, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum((Oid) valueid)); +} + /* ---------- * toast_delete_datum - * @@ -405,10 +443,9 @@ toast_delete_datum(Relation rel, Datum value, bool is_speculative) /* * Setup a scan key to find chunks with matching va_valueid */ - ScanKeyInit(&toastkey, - (AttrNumber) 1, - BTEqualStrategyNumber, F_OIDEQ, - ObjectIdGetDatum(toast_pointer.va_valueid)); + toast_valueid_scankey_init(&toastkey, + TupleDescAttr(toastrel->rd_att, 0)->atttypid, + toast_pointer.va_valueid); /* * Find all the chunks. (We don't actually care whether we see them in @@ -455,6 +492,7 @@ toastrel_valueid_exists(Relation toastrel, Oid8 valueid) int num_indexes; int validIndex; Relation *toastidxs; + Oid toast_typid; /* Fetch a valid index relation */ validIndex = toast_open_indexes(toastrel, @@ -462,13 +500,12 @@ toastrel_valueid_exists(Relation toastrel, Oid8 valueid) &toastidxs, &num_indexes); + toast_typid = TupleDescAttr(toastrel->rd_att, 0)->atttypid; + /* * Setup a scan key to find chunks with matching va_valueid */ - ScanKeyInit(&toastkey, - (AttrNumber) 1, - BTEqualStrategyNumber, F_OIDEQ, - ObjectIdGetDatum(valueid)); + toast_valueid_scankey_init(&toastkey, toast_typid, valueid); /* * Is there any such chunk? diff --git a/src/backend/access/heap/heaptoast.c b/src/backend/access/heap/heaptoast.c index 81154c17376c..81d5085ca398 100644 --- a/src/backend/access/heap/heaptoast.c +++ b/src/backend/access/heap/heaptoast.c @@ -640,6 +640,7 @@ heap_fetch_toast_slice(Relation toastrel, Oid8 valueid, int32 attrsize, int num_indexes; int validIndex; int32 max_chunk_size; + Oid toast_typid; /* Look for the valid index of toast relation */ validIndex = toast_open_indexes(toastrel, @@ -647,6 +648,7 @@ heap_fetch_toast_slice(Relation toastrel, Oid8 valueid, int32 attrsize, &toastidxs, &num_indexes); + toast_typid = TupleDescAttr(toastrel->rd_att, 0)->atttypid; max_chunk_size = TOAST_OID_MAX_CHUNK_SIZE; totalchunks = ((attrsize - 1) / max_chunk_size) + 1; @@ -655,10 +657,7 @@ heap_fetch_toast_slice(Relation toastrel, Oid8 valueid, int32 attrsize, Assert(endchunk <= totalchunks); /* Set up a scan key to fetch from the index. */ - ScanKeyInit(&toastkey[0], - (AttrNumber) 1, - BTEqualStrategyNumber, F_OIDEQ, - ObjectIdGetDatum(valueid)); + toast_valueid_scankey_init(&toastkey[0], toast_typid, valueid); /* * No additional condition if fetching all chunks. Otherwise, use an diff --git a/src/backend/catalog/toasting.c b/src/backend/catalog/toasting.c index a4a7b7c91b31..a7079661666c 100644 --- a/src/backend/catalog/toasting.c +++ b/src/backend/catalog/toasting.c @@ -32,6 +32,7 @@ #include "nodes/makefuncs.h" #include "utils/fmgroids.h" #include "utils/rel.h" +#include "utils/lsyscache.h" #include "utils/syscache.h" static void CheckAndCreateToastTable(Oid relOid, Datum reloptions, @@ -173,6 +174,9 @@ create_toast_table(Relation rel, Oid toastOid, Oid toastIndexOid, case STDRD_OPTION_TOAST_VALUE_TYPE_OID: toast_chunkid_typid = OIDOID; break; + case STDRD_OPTION_TOAST_VALUE_TYPE_OID8: + toast_chunkid_typid = OID8OID; + break; case STDRD_OPTION_TOAST_VALUE_TYPE_INVALID: elog(ERROR, "unexpected toast_value_type value %d", value_type); @@ -210,7 +214,8 @@ create_toast_table(Relation rel, Oid toastOid, Oid toastIndexOid, ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("toast chunk_id type not set while in binary upgrade mode"))); - if (binary_upgrade_next_toast_chunk_id_typoid != OIDOID) + if (binary_upgrade_next_toast_chunk_id_typoid != OIDOID && + binary_upgrade_next_toast_chunk_id_typoid != OID8OID) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("cannot support toast chunk_id type %u in binary upgrade mode", @@ -235,6 +240,19 @@ create_toast_table(Relation rel, Oid toastOid, Oid toastIndexOid, snprintf(toast_idxname, sizeof(toast_idxname), "pg_toast_%u_index", relOid); + /* + * Special case here. If OIDOldToast is defined, we need to rely on the + * existing table for the job because we do not want to create an + * inconsistent relation that would conflict with the parent and break + * the world. + */ + if (OidIsValid(OIDOldToast)) + { + toast_chunkid_typid = get_atttype(OIDOldToast, 1); + if (!OidIsValid(toast_chunkid_typid)) + elog(ERROR, "cache lookup failed for relation %u", OIDOldToast); + } + /* this is pretty painful... need a tuple descriptor */ tupdesc = CreateTemplateTupleDesc(3); TupleDescInitEntry(tupdesc, (AttrNumber) 1, @@ -353,7 +371,10 @@ create_toast_table(Relation rel, Oid toastOid, Oid toastIndexOid, collationIds[0] = InvalidOid; collationIds[1] = InvalidOid; - opclassIds[0] = OID_BTREE_OPS_OID; + if (toast_chunkid_typid == OID8OID) + opclassIds[0] = OID8_BTREE_OPS_OID; + else + opclassIds[0] = OID_BTREE_OPS_OID; opclassIds[1] = INT4_BTREE_OPS_OID; coloptions[0] = 0; diff --git a/src/backend/replication/logical/reorderbuffer.c b/src/backend/replication/logical/reorderbuffer.c index 534fe338b386..dd4b6e8f7f45 100644 --- a/src/backend/replication/logical/reorderbuffer.c +++ b/src/backend/replication/logical/reorderbuffer.c @@ -5036,6 +5036,8 @@ ReorderBufferToastAppendChunk(ReorderBuffer *rb, ReorderBufferTXN *txn, TupleDesc desc = RelationGetDescr(relation); Oid8 chunk_id; int32 chunk_seq; + Oid valueid_type; + Datum valueid_datum; if (txn->toast_hash == NULL) ReorderBufferToastInitHash(rb, txn); @@ -5043,7 +5045,12 @@ ReorderBufferToastAppendChunk(ReorderBuffer *rb, ReorderBufferTXN *txn, Assert(IsToastRelation(relation)); newtup = change->data.tp.newtuple; - chunk_id = DatumGetObjectId(fastgetattr(newtup, 1, desc, &isnull)); + valueid_type = TupleDescAttr(desc, 0)->atttypid; + valueid_datum = fastgetattr(newtup, 1, desc, &isnull); + if (valueid_type == OID8OID) + chunk_id = DatumGetObjectId8(valueid_datum); + else + chunk_id = DatumGetObjectId(valueid_datum); Assert(!isnull); chunk_seq = DatumGetInt32(fastgetattr(newtup, 2, desc, &isnull)); Assert(!isnull); diff --git a/src/test/regress/expected/reloptions.out b/src/test/regress/expected/reloptions.out index 9b4c8587aa49..95e4528f7793 100644 --- a/src/test/regress/expected/reloptions.out +++ b/src/test/regress/expected/reloptions.out @@ -199,7 +199,7 @@ CREATE TABLE reloptions_test2 (s VARCHAR) WITH (toast.toast_value_type = 'oid'); ERROR: unrecognized parameter "toast_value_type" CREATE TABLE reloptions_test2 (s VARCHAR) WITH (toast_value_type = 'int8'); ERROR: invalid value for enum option "toast_value_type": int8 -DETAIL: Valid values are "oid". +DETAIL: Valid values are "oid" and "oid8". CREATE TABLE reloptions_test2 (s VARCHAR) WITH (toast_value_type = 'oid'); SELECT reloptions FROM pg_class WHERE oid = 'reloptions_test2'::regclass; reloptions @@ -207,6 +207,19 @@ SELECT reloptions FROM pg_class WHERE oid = 'reloptions_test2'::regclass; {toast_value_type=oid} (1 row) +-- The option is only consulted when the TOAST relation is created, so +-- changing it afterwards leaves chunk_id alone. +ALTER TABLE reloptions_test2 SET (toast_value_type = 'oid8'); +SELECT a.atttypid::regtype AS chunk_id_type + FROM pg_class AS c, pg_attribute AS a + WHERE c.oid = 'reloptions_test2'::regclass AND + a.attrelid = c.reltoastrelid AND a.attname = 'chunk_id'; + chunk_id_type +--------------- + oid +(1 row) + +ALTER TABLE reloptions_test2 RESET (toast_value_type); DROP TABLE reloptions_test2; -- relkinds not supported. CREATE INDEX reloptions_test_idx0 ON reloptions_test (s) diff --git a/src/test/regress/sql/reloptions.sql b/src/test/regress/sql/reloptions.sql index 503d15d16e51..80939ee8b5bd 100644 --- a/src/test/regress/sql/reloptions.sql +++ b/src/test/regress/sql/reloptions.sql @@ -117,6 +117,14 @@ CREATE TABLE reloptions_test2 (s VARCHAR) WITH (toast.toast_value_type = 'oid'); CREATE TABLE reloptions_test2 (s VARCHAR) WITH (toast_value_type = 'int8'); CREATE TABLE reloptions_test2 (s VARCHAR) WITH (toast_value_type = 'oid'); SELECT reloptions FROM pg_class WHERE oid = 'reloptions_test2'::regclass; +-- The option is only consulted when the TOAST relation is created, so +-- changing it afterwards leaves chunk_id alone. +ALTER TABLE reloptions_test2 SET (toast_value_type = 'oid8'); +SELECT a.atttypid::regtype AS chunk_id_type + FROM pg_class AS c, pg_attribute AS a + WHERE c.oid = 'reloptions_test2'::regclass AND + a.attrelid = c.reltoastrelid AND a.attname = 'chunk_id'; +ALTER TABLE reloptions_test2 RESET (toast_value_type); DROP TABLE reloptions_test2; -- relkinds not supported. CREATE INDEX reloptions_test_idx0 ON reloptions_test (s) diff --git a/doc/src/sgml/ref/create_table.sgml b/doc/src/sgml/ref/create_table.sgml index 79622c1a23d6..6f9a87ef449f 100644 --- a/doc/src/sgml/ref/create_table.sgml +++ b/doc/src/sgml/ref/create_table.sgml @@ -1663,7 +1663,10 @@ WITH ( MODULUS numeric_literal, REM Specifies the attribute type of chunk_id to use when creating a TOAST relation for this table. The - default is oid. + default is oid; oid8 assigns + oid8 instead, which allows a larger number of distinct + TOASTed values at the price of four extra bytes per + out-of-line pointer. This parameter cannot be set for TOAST tables. diff --git a/doc/src/sgml/storage.sgml b/doc/src/sgml/storage.sgml index 90aae3defcbc..21bd67c11ea0 100644 --- a/doc/src/sgml/storage.sgml +++ b/doc/src/sgml/storage.sgml @@ -421,14 +421,15 @@ most TOAST_OID_MAX_CHUNK_SIZE bytes (by default this value is c so that four chunk rows will fit on a page, making it about 2000 bytes). Each chunk is stored as a separate row in the TOAST table belonging to the owning table. Every -TOAST table has the columns chunk_id (an OID -identifying the particular TOASTed value), +TOAST table has the columns +chunk_id (an OID or an OID8 identifying +the particular TOASTed value), chunk_seq (a sequence number for the chunk within its value), and chunk_data (the actual data of the chunk). A unique index on chunk_id and chunk_seq provides fast retrieval of the values. A pointer datum representing an out-of-line on-disk TOASTed value therefore needs to store the OID of the -TOAST table in which to look and the OID of the specific value +TOAST table in which to look and the specific value (its chunk_id). For convenience, pointer datums also store the logical datum size (original uncompressed data length), physical stored size (different if compression was applied), and the compression method used, if diff --git a/contrib/amcheck/verify_heapam.c b/contrib/amcheck/verify_heapam.c index 33b00fa4dc5c..56167ecfe1d0 100644 --- a/contrib/amcheck/verify_heapam.c +++ b/contrib/amcheck/verify_heapam.c @@ -28,7 +28,6 @@ #include "storage/procarray.h" #include "storage/read_stream.h" #include "utils/builtins.h" -#include "utils/fmgroids.h" #include "utils/rel.h" #include "utils/tuplestore.h" @@ -1876,8 +1875,12 @@ check_toasted_attribute(HeapCheckContext *ctx, ToastedAttribute *ta) uint32 extsize; int32 expected_chunk_seq = 0; int32 last_chunk_seq; - int32 max_chunk_size = TOAST_OID_MAX_CHUNK_SIZE; + int32 max_chunk_size; Oid8 toast_valueid; + Oid toast_typid; + + toast_typid = TupleDescAttr(ctx->toast_rel->rd_att, 0)->atttypid; + max_chunk_size = TOAST_OID_MAX_CHUNK_SIZE; extsize = VARATT_EXTERNAL_OID_GET_EXTSIZE(ta->toast_pointer); last_chunk_seq = (extsize - 1) / max_chunk_size; @@ -1885,10 +1888,8 @@ check_toasted_attribute(HeapCheckContext *ctx, ToastedAttribute *ta) /* * Setup a scan key to find chunks in toast table with matching va_valueid */ - ScanKeyInit(&toastkey, - (AttrNumber) 1, - BTEqualStrategyNumber, F_OIDEQ, - ObjectIdGetDatum(ta->toast_pointer.va_valueid)); + toast_valueid_scankey_init(&toastkey, toast_typid, + ta->toast_pointer.va_valueid); /* * Check if any chunks for this toasted object exist in the toast table, -- 2.55.0