From 392fe1b61eb7dbe84d4ad65f6749e74d075d370f Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Tue, 8 Jul 2025 08:20:31 +0900 Subject: [PATCH v2 12/13] Add new vartag_external for 8-byte TOAST values This is a new type of external TOAST pointer, able to be fed 8-byte TOAST values. It uses a dedicated vartag_external, which is used when a TOAST table uses bigint for its chunk_id. The relevant callbacks are added to toast_external.c. --- src/include/access/heaptoast.h | 8 +- src/include/varatt.h | 31 ++++- src/backend/access/common/toast_external.c | 124 ++++++++++++++++++ src/backend/access/common/toast_internals.c | 11 +- src/backend/access/heap/heaptoast.c | 69 ++++++++-- .../replication/logical/reorderbuffer.c | 10 +- doc/src/sgml/storage.sgml | 6 +- contrib/amcheck/verify_heapam.c | 2 +- 8 files changed, 239 insertions(+), 22 deletions(-) diff --git a/src/include/access/heaptoast.h b/src/include/access/heaptoast.h index 673e96f5488c..a39ad79a5ae9 100644 --- a/src/include/access/heaptoast.h +++ b/src/include/access/heaptoast.h @@ -81,6 +81,12 @@ #define EXTERN_TUPLE_MAX_SIZE MaximumBytesPerTuple(EXTERN_TUPLES_PER_PAGE) +#define TOAST_MAX_CHUNK_SIZE_INT8 \ + (EXTERN_TUPLE_MAX_SIZE - \ + MAXALIGN(SizeofHeapTupleHeader) - \ + (sizeof(uint32) * 2) - \ + sizeof(int32) - \ + VARHDRSZ) #define TOAST_MAX_CHUNK_SIZE_OID \ (EXTERN_TUPLE_MAX_SIZE - \ MAXALIGN(SizeofHeapTupleHeader) - \ @@ -89,7 +95,7 @@ VARHDRSZ) /* Maximum size of chunk possible */ -#define TOAST_MAX_CHUNK_SIZE TOAST_MAX_CHUNK_SIZE_OID +#define TOAST_MAX_CHUNK_SIZE Max(TOAST_MAX_CHUNK_SIZE_INT8, TOAST_MAX_CHUNK_SIZE_OID) /* ---------- * heap_toast_insert_or_update - diff --git a/src/include/varatt.h b/src/include/varatt.h index 793030dae932..aa36e8e1f561 100644 --- a/src/include/varatt.h +++ b/src/include/varatt.h @@ -41,6 +41,29 @@ typedef struct varatt_external_oid Oid va_toastrelid; /* RelID of TOAST table containing it */ } varatt_external_oid; +/* + * struct varatt_external_int8 is a "larger" version of "TOAST pointer", + * that uses an 8-byte integer as value. + * + * This follows the same properties as varatt_external_oid, except that + * this is used in TOAST relations with int8 as attribute for chunk_id. + */ +typedef struct varatt_external_int8 +{ + int32 va_rawsize; /* Original data size (includes header) */ + uint32 va_extinfo; /* External saved size (without header) and + * compression method */ + /* + * Unique ID of value within TOAST table, as two uint32 for alignment + * and padding. + * XXX: think for example about the addition of an extra field for + * meta-data and/or more compression data, even if it's OK here). + */ + uint32 va_valueid_lo; + uint32 va_valueid_hi; + Oid va_toastrelid; /* RelID of TOAST table containing it */ +} varatt_external_int8; + /* * These macros define the "saved size" portion of va_extinfo. Its remaining @@ -91,6 +114,7 @@ typedef enum vartag_external VARTAG_INDIRECT = 1, VARTAG_EXPANDED_RO = 2, VARTAG_EXPANDED_RW = 3, + VARTAG_ONDISK_INT8 = 4, VARTAG_ONDISK_OID = 18 } vartag_external; @@ -102,6 +126,7 @@ typedef enum vartag_external ((tag) == VARTAG_INDIRECT ? sizeof(varatt_indirect) : \ VARTAG_IS_EXPANDED(tag) ? sizeof(varatt_expanded) : \ (tag) == VARTAG_ONDISK_OID ? sizeof(varatt_external_oid) : \ + (tag) == VARTAG_ONDISK_INT8 ? sizeof(varatt_external_int8) : \ (AssertMacro(false), 0)) /* @@ -294,8 +319,10 @@ typedef struct #define VARATT_IS_EXTERNAL(PTR) VARATT_IS_1B_E(PTR) #define VARATT_IS_EXTERNAL_ONDISK_OID(PTR) \ (VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_ONDISK_OID) +#define VARATT_IS_EXTERNAL_ONDISK_INT8(PTR) \ + (VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_ONDISK_INT8) #define VARATT_IS_EXTERNAL_ONDISK(PTR) \ - (VARATT_IS_EXTERNAL_ONDISK_OID(PTR)) + (VARATT_IS_EXTERNAL_ONDISK_OID(PTR) || VARATT_IS_EXTERNAL_ONDISK_INT8(PTR)) #define VARATT_IS_EXTERNAL_INDIRECT(PTR) \ (VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_INDIRECT) #define VARATT_IS_EXTERNAL_EXPANDED_RO(PTR) \ @@ -339,7 +366,7 @@ typedef struct /* * Same for external Datums; but note argument is a struct - * varatt_external_oid. + * varatt_external_oid or varatt_external_int8. */ #define VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer) \ ((toast_pointer).va_extinfo & VARLENA_EXTSIZE_MASK) diff --git a/src/backend/access/common/toast_external.c b/src/backend/access/common/toast_external.c index d179f0143803..0e79ac8acae8 100644 --- a/src/backend/access/common/toast_external.c +++ b/src/backend/access/common/toast_external.c @@ -14,9 +14,22 @@ #include "postgres.h" #include "access/detoast.h" +#include "access/genam.h" #include "access/heaptoast.h" +#include "access/toast_counter.h" #include "access/toast_external.h" #include "catalog/catalog.h" +#include "miscadmin.h" +#include "utils/fmgroids.h" +#include "utils/snapmgr.h" + + +/* Callbacks for VARTAG_ONDISK_INT8 */ +static void ondisk_int8_to_external_data(struct varlena *attr, + toast_external_data *data); +static struct varlena *ondisk_int8_create_external_data(toast_external_data data); +static uint64 ondisk_int8_get_new_value(Relation toastrel, Oid indexid, + AttrNumber attnum); /* Callbacks for VARTAG_ONDISK_OID */ static void ondisk_oid_to_external_data(struct varlena *attr, @@ -26,6 +39,12 @@ static uint64 ondisk_oid_get_new_value(Relation toastrel, Oid indexid, AttrNumber attnum); +/* + * Size of an EXTERNAL datum that contains a standard TOAST pointer + * (int8 value). + */ +#define TOAST_POINTER_INT8_SIZE (VARHDRSZ_EXTERNAL + sizeof(varatt_external_int8)) + /* * Size of an EXTERNAL datum that contains a standard TOAST pointer (OID * value). @@ -46,6 +65,13 @@ static uint64 ondisk_oid_get_new_value(Relation toastrel, Oid indexid, * individual fields. */ static const toast_external_info toast_external_infos[TOAST_EXTERNAL_INFO_SIZE] = { + [VARTAG_ONDISK_INT8] = { + .toast_pointer_size = TOAST_POINTER_INT8_SIZE, + .maximum_chunk_size = TOAST_MAX_CHUNK_SIZE_INT8, + .to_external_data = ondisk_int8_to_external_data, + .create_external_data = ondisk_int8_create_external_data, + .get_new_value = ondisk_int8_get_new_value, + }, [VARTAG_ONDISK_OID] = { .toast_pointer_size = TOAST_POINTER_OID_SIZE, .maximum_chunk_size = TOAST_MAX_CHUNK_SIZE_OID, @@ -78,6 +104,104 @@ toast_external_info_get_pointer_size(uint8 tag) * the in-memory representation toast_external_data used in the backend. */ +/* Callbacks for VARTAG_ONDISK_INT8 */ +static void +ondisk_int8_to_external_data(struct varlena *attr, toast_external_data *data) +{ + varatt_external_int8 external; + + VARATT_EXTERNAL_GET_POINTER(external, attr); + data->rawsize = external.va_rawsize; + + /* External size and compression methods are stored in the same field */ + if (VARATT_EXTERNAL_IS_COMPRESSED(external)) + { + data->extsize = VARATT_EXTERNAL_GET_EXTSIZE(external); + data->compression_method = VARATT_EXTERNAL_GET_COMPRESS_METHOD(external); + } + else + { + data->extsize = external.va_extinfo; + data->compression_method = TOAST_INVALID_COMPRESSION_ID; + } + + data->value = (((uint64) external.va_valueid_hi) << 32) | + external.va_valueid_lo; + data->toastrelid = external.va_toastrelid; + +} + +static struct varlena * +ondisk_int8_create_external_data(toast_external_data data) +{ + struct varlena *result = NULL; + varatt_external_int8 external; + + external.va_rawsize = data.rawsize; + + if (data.compression_method != TOAST_INVALID_COMPRESSION_ID) + { + /* Set size and compression method, in a single field. */ + VARATT_EXTERNAL_SET_SIZE_AND_COMPRESS_METHOD(external, + data.extsize, + data.compression_method); + } + else + external.va_extinfo = data.extsize; + + external.va_toastrelid = data.toastrelid; + external.va_valueid_hi = (((uint64) data.value) >> 32); + external.va_valueid_lo = (uint32) data.value; + + result = (struct varlena *) palloc(TOAST_POINTER_INT8_SIZE); + SET_VARTAG_EXTERNAL(result, VARTAG_ONDISK_INT8); + memcpy(VARDATA_EXTERNAL(result), &external, sizeof(external)); + + return result; +} + +static uint64 +ondisk_int8_get_new_value(Relation toastrel, Oid indexid, + AttrNumber attnum) +{ + uint64 new_value; + SysScanDesc scan; + ScanKeyData key; + bool collides = false; + +retry: + new_value = GetNewToastId(); + + /* No indexes in bootstrap mode, so leave */ + if (IsBootstrapProcessingMode()) + return new_value; + + Assert(IsSystemRelation(toastrel)); + + CHECK_FOR_INTERRUPTS(); + + /* + * Check if the new value picked already exists in the toast relation. + * If there is a conflict, retry. + */ + ScanKeyInit(&key, + attnum, + BTEqualStrategyNumber, F_INT8EQ, + Int64GetDatum(new_value)); + + /* see notes in GetNewOidWithIndex() above about using SnapshotAny */ + scan = systable_beginscan(toastrel, indexid, true, + SnapshotAny, 1, &key); + collides = HeapTupleIsValid(systable_getnext(scan)); + systable_endscan(scan); + + if (collides) + goto retry; + + return new_value; +} + + /* Callbacks for VARTAG_ONDISK_OID */ static void ondisk_oid_to_external_data(struct varlena *attr, toast_external_data *data) diff --git a/src/backend/access/common/toast_internals.c b/src/backend/access/common/toast_internals.c index 28e2867f8209..c6b2d1522cef 100644 --- a/src/backend/access/common/toast_internals.c +++ b/src/backend/access/common/toast_internals.c @@ -18,7 +18,6 @@ #include "access/heapam.h" #include "access/heaptoast.h" #include "access/table.h" -#include "access/toast_counter.h" #include "access/toast_external.h" #include "access/toast_internals.h" #include "access/xact.h" @@ -158,6 +157,8 @@ toast_save_datum(Relation rel, Datum value, */ toastrel = table_open(rel->rd_rel->reltoastrelid, RowExclusiveLock); toasttupDesc = toastrel->rd_att; + toast_typid = TupleDescAttr(toasttupDesc, 0)->atttypid; + Assert(toast_typid == OIDOID || toast_typid == INT8OID); /* * Grab the information for toast_external_data. @@ -165,12 +166,12 @@ toast_save_datum(Relation rel, Datum value, * Note: if we support multiple external vartags for a single value * type, we would need to be smarter in the vartag selection. */ - tag = VARTAG_ONDISK_OID; + if (toast_typid == OIDOID) + tag = VARTAG_ONDISK_OID; + else if (toast_typid == INT8OID) + tag = VARTAG_ONDISK_INT8; info = toast_external_get_info(tag); - toast_typid = TupleDescAttr(toasttupDesc, 0)->atttypid; - Assert(toast_typid == OIDOID || toast_typid == INT8OID); - /* Open all the toast indexes and look for the valid one */ validIndex = toast_open_indexes(toastrel, RowExclusiveLock, diff --git a/src/backend/access/heap/heaptoast.c b/src/backend/access/heap/heaptoast.c index 7a54cf7fae34..2121eb02848e 100644 --- a/src/backend/access/heap/heaptoast.c +++ b/src/backend/access/heap/heaptoast.c @@ -31,7 +31,9 @@ #include "access/toast_external.h" #include "access/toast_helper.h" #include "access/toast_internals.h" +#include "access/toast_type.h" #include "utils/fmgroids.h" +#include "utils/syscache.h" /* ---------- @@ -145,12 +147,52 @@ heap_toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, /* * Retrieve the toast pointer size based on the type of external TOAST * pointer assumed to be used. - * - * Only one format of external TOAST pointer is supported currently, - * making this code simple, based on a single vartag. */ - ttc.ttc_toast_pointer_size = - toast_external_info_get_pointer_size(VARTAG_ONDISK_OID); + if (OidIsValid(rel->rd_rel->reltoastrelid)) + { + HeapTuple atttuple; + Form_pg_attribute atttoast; + uint8 vartag; + + /* + * XXX: This is very unlikely efficient, but it is not possible to + * rely on the relation cache to retrieve this information as syscache + * lookups should not happen when loading critical entries. + */ + atttuple = SearchSysCacheAttNum(rel->rd_rel->reltoastrelid, 1); + if (!HeapTupleIsValid(atttuple)) + elog(ERROR, "cache lookup failed for relation %u", + rel->rd_rel->reltoastrelid); + atttoast = (Form_pg_attribute) GETSTRUCT(atttuple); + + if (atttoast->atttypid == OIDOID) + vartag = VARTAG_ONDISK_OID; + else if (atttoast->atttypid == INT8OID) + vartag = VARTAG_ONDISK_INT8; + else + Assert(false); + ttc.ttc_toast_pointer_size = + toast_external_info_get_pointer_size(vartag); + ReleaseSysCache(atttuple); + } + else + { + /* + * No TOAST relation to rely on, which is a case possible when + * dealing with partitioned tables, for example. Hence, do a best + * guess based on the GUC default_toast_type. + */ + uint8 vartag = VARTAG_ONDISK_OID; + + if (default_toast_type == TOAST_TYPE_INT8) + vartag = VARTAG_ONDISK_INT8; + else if (default_toast_type == TOAST_TYPE_OID) + vartag = VARTAG_ONDISK_OID; + else + Assert(false); + ttc.ttc_toast_pointer_size = + toast_external_info_get_pointer_size(vartag); + } ttc.ttc_rel = rel; ttc.ttc_values = toast_values; @@ -654,7 +696,7 @@ heap_fetch_toast_slice(Relation toastrel, uint64 valueid, int32 attrsize, int32 max_chunk_size; const toast_external_info *info; uint8 tag = VARTAG_INDIRECT; /* init value does not matter */ - Oid toast_typid; + Oid toast_typid = InvalidOid; /* Look for the valid index of toast relation */ validIndex = toast_open_indexes(toastrel, @@ -673,14 +715,19 @@ heap_fetch_toast_slice(Relation toastrel, uint64 valueid, int32 attrsize, * to pass down this information from the upper callers. This is * currently on required for the maximum chunk_size. */ - tag = VARTAG_ONDISK_OID; - info = toast_external_get_info(tag); - - max_chunk_size = info->maximum_chunk_size; - toast_typid = TupleDescAttr(toastrel->rd_att, 0)->atttypid; Assert(toast_typid == OIDOID || toast_typid == INT8OID); + if (toast_typid == OIDOID) + tag = VARTAG_ONDISK_OID; + else if (toast_typid == INT8OID) + tag = VARTAG_ONDISK_INT8; + else + Assert(false); + + info = toast_external_get_info(tag); + max_chunk_size = info->maximum_chunk_size; + totalchunks = ((attrsize - 1) / max_chunk_size) + 1; startchunk = sliceoffset / max_chunk_size; endchunk = (sliceoffset + slicelength - 1) / max_chunk_size; diff --git a/src/backend/replication/logical/reorderbuffer.c b/src/backend/replication/logical/reorderbuffer.c index 77939c7f849c..834040395e62 100644 --- a/src/backend/replication/logical/reorderbuffer.c +++ b/src/backend/replication/logical/reorderbuffer.c @@ -4971,14 +4971,22 @@ ReorderBufferToastAppendChunk(ReorderBuffer *rb, ReorderBufferTXN *txn, TupleDesc desc = RelationGetDescr(relation); uint64 chunk_id; int32 chunk_seq; + Oid toast_typid; if (txn->toast_hash == NULL) ReorderBufferToastInitHash(rb, txn); + toast_typid = TupleDescAttr(desc, 0)->atttypid; Assert(IsToastRelation(relation)); newtup = change->data.tp.newtuple; - chunk_id = DatumGetObjectId(fastgetattr(newtup, 1, desc, &isnull)); + /* This depends on the type of TOAST value dealt with. */ + if (toast_typid == OIDOID) + chunk_id = DatumGetObjectId(fastgetattr(newtup, 1, desc, &isnull)); + else if (toast_typid == INT8OID) + chunk_id = DatumGetUInt64(fastgetattr(newtup, 1, desc, &isnull)); + else + Assert(false); Assert(!isnull); chunk_seq = DatumGetInt32(fastgetattr(newtup, 2, desc, &isnull)); Assert(!isnull); diff --git a/doc/src/sgml/storage.sgml b/doc/src/sgml/storage.sgml index 564783a1c559..29ba80e8423c 100644 --- a/doc/src/sgml/storage.sgml +++ b/doc/src/sgml/storage.sgml @@ -415,7 +415,11 @@ described in more detail below. Out-of-line values are divided (after compression if used) into chunks of at -most TOAST_MAX_CHUNK_SIZE_OID bytes (by default this value is chosen +most TOAST_MAX_CHUNK_SIZE_OID bytes if the +TOAST relation uses the oid type for +chunk_id, or TOAST_MAX_CHUNK_SIZE_INT8 +bytes if the TOAST relation uses the int8 +type for chunk_id (by default these values are chosen so that four chunk rows will fit on a page, making it about 2000 bytes). Each chunk is stored as a separate row in the TOAST table belonging to the owning table. Every diff --git a/contrib/amcheck/verify_heapam.c b/contrib/amcheck/verify_heapam.c index 833811c75437..958b1451b4ff 100644 --- a/contrib/amcheck/verify_heapam.c +++ b/contrib/amcheck/verify_heapam.c @@ -1733,7 +1733,7 @@ check_tuple_attribute(HeapCheckContext *ctx) { uint8 va_tag = VARTAG_EXTERNAL(tp + ctx->offset); - if (va_tag != VARTAG_ONDISK_OID) + if (va_tag != VARTAG_ONDISK_OID && va_tag != VARTAG_ONDISK_INT8) { report_corruption(ctx, psprintf("toasted attribute has unexpected TOAST tag %u", -- 2.50.0