From b429fdf74595bedfa8ff158b96c7462d29db1661 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Thu, 14 Aug 2025 14:10:36 +0900 Subject: [PATCH v5 15/15] Add new vartag_external for 8-byte TOAST values This is a new type of external TOAST pointer, able to be fed 8-byte TOAST values. It uses a dedicated vartag_external, which is used when a TOAST table uses bigint for its chunk_id. The relevant callbacks are added to toast_external.c. --- src/include/access/heaptoast.h | 8 +- src/include/varatt.h | 34 +++- src/backend/access/common/toast_external.c | 145 ++++++++++++++++-- src/backend/access/heap/heaptoast.c | 1 + .../replication/logical/reorderbuffer.c | 10 +- doc/src/sgml/storage.sgml | 6 +- contrib/amcheck/verify_heapam.c | 2 +- 7 files changed, 189 insertions(+), 17 deletions(-) diff --git a/src/include/access/heaptoast.h b/src/include/access/heaptoast.h index afa3d8ca95f7..e944d5f8420c 100644 --- a/src/include/access/heaptoast.h +++ b/src/include/access/heaptoast.h @@ -81,6 +81,12 @@ #define EXTERN_TUPLE_MAX_SIZE MaximumBytesPerTuple(EXTERN_TUPLES_PER_PAGE) +#define TOAST_OID8_MAX_CHUNK_SIZE \ + (EXTERN_TUPLE_MAX_SIZE - \ + MAXALIGN(SizeofHeapTupleHeader) - \ + (sizeof(uint32) * 2) - \ + sizeof(int32) - \ + VARHDRSZ) #define TOAST_OID_MAX_CHUNK_SIZE \ (EXTERN_TUPLE_MAX_SIZE - \ MAXALIGN(SizeofHeapTupleHeader) - \ @@ -89,7 +95,7 @@ VARHDRSZ) /* Maximum size of chunk possible */ -#define TOAST_MAX_CHUNK_SIZE TOAST_OID_MAX_CHUNK_SIZE +#define TOAST_MAX_CHUNK_SIZE Max(TOAST_OID_MAX_CHUNK_SIZE, TOAST_OID8_MAX_CHUNK_SIZE) /* ---------- * heap_toast_insert_or_update - diff --git a/src/include/varatt.h b/src/include/varatt.h index 035c0f95e5b6..de38d1cd1ce1 100644 --- a/src/include/varatt.h +++ b/src/include/varatt.h @@ -41,6 +41,27 @@ typedef struct varatt_external_oid Oid va_toastrelid; /* RelID of TOAST table containing it */ } varatt_external_oid; +/* + * struct varatt_external_oid8 is a "larger" version of "TOAST pointer", + * that uses an 8-byte integer as value. + * + * This follows the same properties as varatt_external_oid, except that + * this is used in TOAST relations with oid8 as attribute for chunk_id. + */ +typedef struct varatt_external_oid8 +{ + int32 va_rawsize; /* Original data size (includes header) */ + uint32 va_extinfo; /* External saved size (without header) and + * compression method */ + /* + * Unique ID of value within TOAST table, as two uint32 for alignment + * and padding. + */ + uint32 va_valueid_lo; + uint32 va_valueid_hi; + Oid va_toastrelid; /* RelID of TOAST table containing it */ +} varatt_external_oid8; + /* * These macros define the "saved size" portion of va_extinfo. Its remaining * two high-order bits identify the compression method. @@ -90,6 +111,7 @@ typedef enum vartag_external VARTAG_INDIRECT = 1, VARTAG_EXPANDED_RO = 2, VARTAG_EXPANDED_RW = 3, + VARTAG_ONDISK_OID8 = 4, VARTAG_ONDISK_OID = 18 } vartag_external; @@ -111,6 +133,8 @@ VARTAG_SIZE(vartag_external tag) return sizeof(varatt_expanded); else if (tag == VARTAG_ONDISK_OID) return sizeof(varatt_external_oid); + else if (tag == VARTAG_ONDISK_OID8) + return sizeof(varatt_external_oid8); else { Assert(false); @@ -367,11 +391,19 @@ VARATT_IS_EXTERNAL_ONDISK_OID(const void *PTR) return VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_ONDISK_OID; } +/* Is varlena datum a pointer to on-disk toasted data with OID8 value? */ +static inline bool +VARATT_IS_EXTERNAL_ONDISK_OID8(const void *PTR) +{ + return VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_ONDISK_OID8; +} + /* Is varlena datum a pointer to on-disk toasted data? */ static inline bool VARATT_IS_EXTERNAL_ONDISK(const void *PTR) { - return VARATT_IS_EXTERNAL_ONDISK_OID(PTR); + return VARATT_IS_EXTERNAL_ONDISK_OID(PTR) || + VARATT_IS_EXTERNAL_ONDISK_OID8(PTR); } /* Is varlena datum an indirect pointer? */ diff --git a/src/backend/access/common/toast_external.c b/src/backend/access/common/toast_external.c index 8f58195051cf..f0f718085e8d 100644 --- a/src/backend/access/common/toast_external.c +++ b/src/backend/access/common/toast_external.c @@ -18,8 +18,19 @@ #include "postgres.h" #include "access/detoast.h" +#include "access/genam.h" #include "access/heaptoast.h" #include "access/toast_external.h" +#include "catalog/catalog.h" +#include "miscadmin.h" +#include "utils/fmgroids.h" +#include "utils/lsyscache.h" + + +/* Callbacks for VARTAG_ONDISK_OID8 */ +static void ondisk_oid8_to_external_data(struct varlena *attr, + toast_external_data *data); +static struct varlena *ondisk_oid8_create_external_data(toast_external_data data); /* Callbacks for VARTAG_ONDISK_OID */ static void ondisk_oid_to_external_data(struct varlena *attr, @@ -28,7 +39,7 @@ static struct varlena *ondisk_oid_create_external_data(toast_external_data data) /* * Fetch the possibly-unaligned contents of an on-disk external TOAST with - * OID values into a local "varatt_external_oid" pointer. + * OID or OID8 values into a local "varatt_external_*" pointer. * * This should be just a memcpy, but some versions of gcc seem to produce * broken code that assumes the datum contents are aligned. Introducing @@ -45,9 +56,20 @@ varatt_external_oid_get_pointer(varatt_external_oid *toast_pointer, memcpy(toast_pointer, VARDATA_EXTERNAL(attre), sizeof(varatt_external_oid)); } +static inline void +varatt_external_oid8_get_pointer(varatt_external_oid8 *toast_pointer, + struct varlena *attr) +{ + varattrib_1b_e *attre = (varattrib_1b_e *) attr; + + Assert(VARATT_IS_EXTERNAL_ONDISK_OID8(attre)); + Assert(VARSIZE_EXTERNAL(attre) == sizeof(varatt_external_oid8) + VARHDRSZ_EXTERNAL); + memcpy(toast_pointer, VARDATA_EXTERNAL(attre), sizeof(varatt_external_oid8)); +} + /* * Decompressed size of an on-disk varlena; but note argument is a struct - * varatt_external_oid. + * varatt_external_oid or varatt_external_oid8. */ static inline Size varatt_external_oid_get_extsize(varatt_external_oid toast_pointer) @@ -55,9 +77,15 @@ varatt_external_oid_get_extsize(varatt_external_oid toast_pointer) return toast_pointer.va_extinfo & VARLENA_EXTSIZE_MASK; } +static inline Size +varatt_external_oid8_get_extsize(varatt_external_oid8 toast_pointer) +{ + return toast_pointer.va_extinfo & VARLENA_EXTSIZE_MASK; +} + /* * Compression method of an on-disk varlena; but note argument is a struct - * varatt_external_oid. + * varatt_external_oid or varatt_external_oid8. */ static inline uint32 varatt_external_oid_get_compress_method(varatt_external_oid toast_pointer) @@ -65,6 +93,12 @@ varatt_external_oid_get_compress_method(varatt_external_oid toast_pointer) return toast_pointer.va_extinfo >> VARLENA_EXTSIZE_BITS; } +static inline uint32 +varatt_external_oid8_get_compress_method(varatt_external_oid8 toast_pointer) +{ + return toast_pointer.va_extinfo >> VARLENA_EXTSIZE_BITS; +} + /* * Testing whether an externally-stored TOAST value is compressed now requires * comparing size stored in va_extinfo (the actual length of the external data) @@ -79,6 +113,19 @@ varatt_external_oid_is_compressed(varatt_external_oid toast_pointer) (Size) (toast_pointer.va_rawsize - VARHDRSZ); } +static inline bool +varatt_external_oid8_is_compressed(varatt_external_oid8 toast_pointer) +{ + return varatt_external_oid8_get_extsize(toast_pointer) < + (Size) (toast_pointer.va_rawsize - VARHDRSZ); +} + +/* + * Size of an EXTERNAL datum that contains a standard TOAST pointer + * (oid8 value). + */ +#define TOAST_POINTER_OID8_SIZE (VARHDRSZ_EXTERNAL + sizeof(varatt_external_oid8)) + /* * Size of an EXTERNAL datum that contains a standard TOAST pointer (OID * value). @@ -99,6 +146,12 @@ varatt_external_oid_is_compressed(varatt_external_oid toast_pointer) * individual fields. */ static const toast_external_info toast_external_infos[TOAST_EXTERNAL_INFO_SIZE] = { + [VARTAG_ONDISK_OID8] = { + .toast_pointer_size = TOAST_POINTER_OID8_SIZE, + .maximum_chunk_size = TOAST_OID_MAX_CHUNK_SIZE, + .to_external_data = ondisk_oid8_to_external_data, + .create_external_data = ondisk_oid8_create_external_data, + }, [VARTAG_ONDISK_OID] = { .toast_pointer_size = TOAST_OID_POINTER_SIZE, .maximum_chunk_size = TOAST_OID_MAX_CHUNK_SIZE, @@ -150,22 +203,33 @@ toast_external_info_get_pointer_size(uint8 tag) uint8 toast_external_assign_vartag(Oid toastrelid, Oid8 valueid) { + Oid toast_typid; + /* - * If dealing with a code path where a TOAST relation may not be assigned, - * like heap_toast_insert_or_update(), just use the legacy - * vartag_external. + * If dealing with a code path where a TOAST relation may not be assigned + * like heap_toast_insert_or_update(), just use the default with an OID + * type. + * + * In bootstrap mode, we should not do any kind of syscache lookups, + * so also rely on OID. */ - if (!OidIsValid(toastrelid)) + if (!OidIsValid(toastrelid) || IsBootstrapProcessingMode()) return VARTAG_ONDISK_OID; /* - * Currently there is only one type of vartag_external supported: 4-byte - * value with OID for the chunk_id type. + * Two types of vartag_external are currently supported: OID and OID8, + * which depend on the type assigned to "chunk_id" for the TOAST table. * - * Note: This routine will be extended to be able to use multiple - * vartag_external within a single TOAST relation type, that may change - * depending on the value used. + * XXX: Should we assign from the start an OID vartag if dealing with + * a TOAST relation with OID8 as value if the value assigned is less + * than UINT_MAX? This just takes the "safe" approach of assigning + * the larger vartag in all cases, but this can be made cheaper + * depending on the OID consumption. */ + toast_typid = get_atttype(toastrelid, 1); + if (toast_typid == OID8OID) + return VARTAG_ONDISK_OID8; + return VARTAG_ONDISK_OID; } @@ -174,6 +238,63 @@ toast_external_assign_vartag(Oid toastrelid, Oid8 valueid) * the in-memory representation toast_external_data used in the backend. */ +/* Callbacks for VARTAG_ONDISK_OID8 */ +static void +ondisk_oid8_to_external_data(struct varlena *attr, toast_external_data *data) +{ + varatt_external_oid8 external; + + varatt_external_oid8_get_pointer(&external, attr); + data->rawsize = external.va_rawsize; + + /* External size and compression methods are stored in the same field */ + if (varatt_external_oid8_is_compressed(external)) + { + data->extsize = varatt_external_oid8_get_extsize(external); + data->compression_method = varatt_external_oid8_get_compress_method(external); + } + else + { + data->extsize = external.va_extinfo; + data->compression_method = TOAST_INVALID_COMPRESSION_ID; + } + + data->valueid = (((uint64) external.va_valueid_hi) << 32) | + external.va_valueid_lo; + data->toastrelid = external.va_toastrelid; + +} + +static struct varlena * +ondisk_oid8_create_external_data(toast_external_data data) +{ + struct varlena *result = NULL; + varatt_external_oid8 external; + + external.va_rawsize = data.rawsize; + + if (data.compression_method != TOAST_INVALID_COMPRESSION_ID) + { + /* Set size and compression method, in a single field. */ + VARATT_EXTERNAL_SET_SIZE_AND_COMPRESS_METHOD(external, + data.extsize, + data.compression_method); + } + else + external.va_extinfo = data.extsize; + + external.va_toastrelid = data.toastrelid; + external.va_valueid_hi = (((uint64) data.valueid) >> 32); + external.va_valueid_lo = (uint32) data.valueid; + + result = (struct varlena *) palloc(TOAST_POINTER_OID8_SIZE); + SET_VARTAG_EXTERNAL(result, VARTAG_ONDISK_OID8); + memcpy(VARDATA_EXTERNAL(result), &external, sizeof(external)); + + return result; +} + + /* Callbacks for VARTAG_ONDISK_OID */ /* diff --git a/src/backend/access/heap/heaptoast.c b/src/backend/access/heap/heaptoast.c index 50e9bf9047f9..cba6e14ea805 100644 --- a/src/backend/access/heap/heaptoast.c +++ b/src/backend/access/heap/heaptoast.c @@ -32,6 +32,7 @@ #include "access/toast_helper.h" #include "access/toast_internals.h" #include "utils/fmgroids.h" +#include "utils/syscache.h" /* ---------- diff --git a/src/backend/replication/logical/reorderbuffer.c b/src/backend/replication/logical/reorderbuffer.c index 9a690a59db2c..f8a207b09157 100644 --- a/src/backend/replication/logical/reorderbuffer.c +++ b/src/backend/replication/logical/reorderbuffer.c @@ -4971,14 +4971,22 @@ ReorderBufferToastAppendChunk(ReorderBuffer *rb, ReorderBufferTXN *txn, TupleDesc desc = RelationGetDescr(relation); Oid8 chunk_id; int32 chunk_seq; + Oid toast_typid; if (txn->toast_hash == NULL) ReorderBufferToastInitHash(rb, txn); + toast_typid = TupleDescAttr(desc, 0)->atttypid; Assert(IsToastRelation(relation)); newtup = change->data.tp.newtuple; - chunk_id = DatumGetObjectId(fastgetattr(newtup, 1, desc, &isnull)); + /* This depends on the type of TOAST value dealt with. */ + if (toast_typid == OIDOID) + chunk_id = DatumGetObjectId(fastgetattr(newtup, 1, desc, &isnull)); + else if (toast_typid == INT8OID) + chunk_id = DatumGetUInt64(fastgetattr(newtup, 1, desc, &isnull)); + else + Assert(false); Assert(!isnull); chunk_seq = DatumGetInt32(fastgetattr(newtup, 2, desc, &isnull)); Assert(!isnull); diff --git a/doc/src/sgml/storage.sgml b/doc/src/sgml/storage.sgml index afddf663fec5..dbec30d48b4a 100644 --- a/doc/src/sgml/storage.sgml +++ b/doc/src/sgml/storage.sgml @@ -417,7 +417,11 @@ described in more detail below. Out-of-line values are divided (after compression if used) into chunks of at -most TOAST_OID_MAX_CHUNK_SIZE bytes (by default this value is chosen +most TOAST_OID_MAX_CHUNK_SIZE bytes if the +TOAST relation uses the oid type for +chunk_id, or TOAST_OID8_MAX_CHUNK_SIZE +bytes if the TOAST relation uses the oid8 +type for chunk_id (by default these values are chosen so that four chunk rows will fit on a page, making it about 2000 bytes). Each chunk is stored as a separate row in the TOAST table belonging to the owning table. Every diff --git a/contrib/amcheck/verify_heapam.c b/contrib/amcheck/verify_heapam.c index 143e6baa35cf..8cea9ad31bcd 100644 --- a/contrib/amcheck/verify_heapam.c +++ b/contrib/amcheck/verify_heapam.c @@ -1733,7 +1733,7 @@ check_tuple_attribute(HeapCheckContext *ctx) { uint8 va_tag = VARTAG_EXTERNAL(tp + ctx->offset); - if (va_tag != VARTAG_ONDISK_OID) + if (va_tag != VARTAG_ONDISK_OID && va_tag != VARTAG_ONDISK_OID8) { report_corruption(ctx, psprintf("toasted attribute has unexpected TOAST tag %u", -- 2.50.0