From 5199347bfc9a3daa45a2058c721167b5d0b1ca1a Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Fri, 11 Sep 2026 15:57:31 +0900 Subject: [PATCH v17 5/5] Add support for TOAST pointers as oid8 This introduces a new varlena external pointer structure, varatt_external_oid8, that carries a 64-bit value ID (Oid8) for TOAST tables using oid8 as their chunk_id type. XXX: Catalog version bump required. --- src/include/access/detoast.h | 49 ++++ src/include/access/heaptoast.h | 22 +- src/include/utils/rel.h | 9 + src/include/utils/relcache.h | 1 + src/include/varatt.h | 78 ++++-- src/backend/access/common/detoast.c | 75 +++--- src/backend/access/common/toast_compression.c | 8 +- src/backend/access/common/toast_internals.c | 245 ++++++++++-------- src/backend/access/heap/heaptoast.c | 2 +- src/backend/access/table/toast_helper.c | 22 +- .../replication/logical/reorderbuffer.c | 15 +- src/backend/utils/adt/varlena.c | 8 +- src/backend/utils/cache/relcache.c | 37 +++ doc/src/sgml/storage.sgml | 12 +- contrib/amcheck/verify_heapam.c | 57 ++-- src/tools/pgindent/typedefs.list | 2 + 16 files changed, 430 insertions(+), 212 deletions(-) diff --git a/src/include/access/detoast.h b/src/include/access/detoast.h index 2b1428b1e81c..e9d8076f77cd 100644 --- a/src/include/access/detoast.h +++ b/src/include/access/detoast.h @@ -12,6 +12,8 @@ #ifndef DETOAST_H #define DETOAST_H +#include "varatt.h" + /* * Macro to fetch the possibly-unaligned contents of an EXTERNAL datum * into a local "varatt_external_oid" toast pointer. This should be @@ -30,9 +32,56 @@ do { \ /* Size of an EXTERNAL datum that contains a standard TOAST pointer */ #define TOAST_OID_POINTER_SIZE (VARHDRSZ_EXTERNAL + sizeof(varatt_external_oid)) +/* Size of an EXTERNAL datum that contains an Oid8 TOAST pointer */ +#define TOAST_OID8_POINTER_SIZE (VARHDRSZ_EXTERNAL + sizeof(varatt_external_oid8)) + /* Size of an EXTERNAL datum that contains an indirection pointer */ #define INDIRECT_POINTER_SIZE (VARHDRSZ_EXTERNAL + sizeof(varatt_indirect)) +/* + * Decoded contents of an on-disk TOAST pointer, independent of whether the + * value ID type. + */ +typedef struct toast_external_data +{ + vartag_external tag; /* VARTAG_ONDISK_* */ + int32 rawsize; /* original data size (includes header) */ + uint32 extinfo; /* saved size + compression method */ + Oid8 valueid; /* value ID (can be widened from Oid) */ + Oid toastrelid; /* OID of the TOAST table containing it */ +} toast_external_data; + +/* + * Decode an on-disk TOAST pointer into a toast_external_data. + */ +static inline void +toast_external_info_get(const struct varlena *attr, toast_external_data *toast_ext_data) +{ + Assert(VARATT_IS_EXTERNAL_ONDISK(attr)); + + toast_ext_data->tag = VARTAG_EXTERNAL(attr); + if (toast_ext_data->tag == VARTAG_ONDISK_OID8) + { + varatt_external_oid8 toast_pointer; + + VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + toast_ext_data->rawsize = toast_pointer.va_rawsize; + toast_ext_data->extinfo = toast_pointer.va_extinfo; + toast_ext_data->valueid = VARATT_EXTERNAL_OID8_GET_VALUEID(toast_pointer); + toast_ext_data->toastrelid = toast_pointer.va_toastrelid; + } + else + { + varatt_external_oid toast_pointer; + + VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + toast_ext_data->rawsize = toast_pointer.va_rawsize; + toast_ext_data->extinfo = toast_pointer.va_extinfo; + toast_ext_data->valueid = toast_pointer.va_valueid; + toast_ext_data->toastrelid = toast_pointer.va_toastrelid; + } +} + /* ---------- * detoast_external_attr() - * diff --git a/src/include/access/heaptoast.h b/src/include/access/heaptoast.h index aaae6e0fef69..4c3fda8a5f9a 100644 --- a/src/include/access/heaptoast.h +++ b/src/include/access/heaptoast.h @@ -69,13 +69,14 @@ /* * When we store an oversize datum externally, we divide it into chunks - * containing at most TOAST_OID_MAX_CHUNK_SIZE data bytes. This number *must* - * be small enough that the completed toast-table tuple (including the - * ID and sequence fields and all overhead) will fit on a page. + * containing at most TOAST_OID_MAX_CHUNK_SIZE or TOAST_OID8_MAX_CHUNK_SIZE + * data bytes, depending on the chunk_id type of the TOAST table. These + * numbers *must* be small enough that the completed toast-table tuple + * (including the ID and sequence fields and all overhead) will fit on a page. * The coding here sets the size on the theory that we want to fit * EXTERN_TUPLES_PER_PAGE tuples of maximum size onto a page. * - * NB: Changing TOAST_OID_MAX_CHUNK_SIZE requires an initdb. + * NB: Changing these values requires an initdb. */ #define EXTERN_TUPLES_PER_PAGE 4 /* tweak only this */ @@ -88,6 +89,19 @@ sizeof(int32) - \ VARHDRSZ) +#define TOAST_OID8_MAX_CHUNK_SIZE \ + (EXTERN_TUPLE_MAX_SIZE - \ + MAXALIGN(SizeofHeapTupleHeader) - \ + sizeof(Oid8) - \ + sizeof(int32) - \ + VARHDRSZ) + +/* + * Select the chunk size for a TOAST table from its value type. + */ +#define TOAST_MAX_CHUNK_SIZE(toast_typid) \ + ((toast_typid) == OID8OID ? TOAST_OID8_MAX_CHUNK_SIZE : TOAST_OID_MAX_CHUNK_SIZE) + /* ---------- * heap_toast_insert_or_update - * diff --git a/src/include/utils/rel.h b/src/include/utils/rel.h index d61432de705e..916b67f5504f 100644 --- a/src/include/utils/rel.h +++ b/src/include/utils/rel.h @@ -250,6 +250,15 @@ typedef struct RelationData */ Oid rd_toastoid; /* Real TOAST table's OID, or InvalidOid */ + /* + * Type OID of the "chunk_id" column of this relation's TOAST table, i.e. + * OIDOID or OID8OID. + * + * This data is filled on demand by RelationGetToastChunkIdType(), not + * at relcache build time, so as to save in syscache lookups. + */ + Oid rd_toastchunkidtype; + bool pgstat_enabled; /* should relation stats be counted */ /* use "struct" here to avoid needing to include pgstat.h: */ struct PgStat_RelationStatus *pgstat_info; /* statistics collection area */ diff --git a/src/include/utils/relcache.h b/src/include/utils/relcache.h index 89c27aa1529f..e17c84908332 100644 --- a/src/include/utils/relcache.h +++ b/src/include/utils/relcache.h @@ -57,6 +57,7 @@ extern List *RelationGetIndexList(Relation relation); extern List *RelationGetStatExtList(Relation relation); extern Oid RelationGetPrimaryKeyIndex(Relation relation, bool deferrable_ok); extern Oid RelationGetReplicaIndex(Relation relation); +extern Oid RelationGetToastChunkIdType(Relation relation); extern List *RelationGetIndexExpressions(Relation relation); extern List *RelationGetDummyIndexExpressions(Relation relation); extern List *RelationGetIndexPredicate(Relation relation); diff --git a/src/include/varatt.h b/src/include/varatt.h index 89b47db08332..77041f7673a9 100644 --- a/src/include/varatt.h +++ b/src/include/varatt.h @@ -42,6 +42,44 @@ StaticAssertDecl((sizeof(int32) + sizeof(uint32) + 2 * sizeof(Oid)) == sizeof(varatt_external_oid), "varatt_external_oid should have no padding"); +/* + * varatt_external_oid8 is a "TOAST pointer" for TOAST tables that use + * Oid8 (64-bit) as their chunk_id type. Same layout as varatt_external_oid + * except for the value ID, which is 8 bytes wide. The value ID is split + * into two uint32 to force alignment. + * + * This struct must not contain any padding, because we sometimes compare + * these pointers using memcmp. + */ +typedef struct varatt_external_oid8 +{ + int32 va_rawsize; /* Original data size (includes header) */ + uint32 va_extinfo; /* External saved size (without header) and + * compression method */ + uint32 va_valueid_lo; /* Low 32 bits of value ID */ + uint32 va_valueid_hi; /* High 32 bits of value ID */ + Oid va_toastrelid; /* RelID of TOAST table containing it */ +} varatt_external_oid8; + +StaticAssertDecl((sizeof(int32) + 3 * sizeof(uint32) + sizeof(Oid)) == + sizeof(varatt_external_oid8), + "varatt_external_oid8 should have no padding"); + +/* Get/set the 64-bit value ID from the lo/hi halves */ +static inline Oid8 +VARATT_EXTERNAL_OID8_GET_VALUEID(varatt_external_oid8 toast_pointer) +{ + return ((Oid8) toast_pointer.va_valueid_lo) | + (((Oid8) toast_pointer.va_valueid_hi) << 32); +} + +static inline void +VARATT_EXTERNAL_OID8_SET_VALUEID(varatt_external_oid8 *toast_pointer, Oid8 id) +{ + toast_pointer->va_valueid_lo = (uint32) id; + toast_pointer->va_valueid_hi = (uint32) (id >> 32); +} + /* * These macros define the "saved size" portion of va_extinfo. Its remaining * two high-order bits identify the compression method. @@ -91,6 +129,7 @@ typedef enum vartag_external VARTAG_INDIRECT = 1, VARTAG_EXPANDED_RO = 2, VARTAG_EXPANDED_RW = 3, + VARTAG_ONDISK_OID8 = 4, VARTAG_ONDISK_OID = 18 } vartag_external; @@ -112,6 +151,8 @@ VARTAG_SIZE(vartag_external tag) return sizeof(varatt_expanded); else if (tag == VARTAG_ONDISK_OID) return sizeof(varatt_external_oid); + else if (tag == VARTAG_ONDISK_OID8) + return sizeof(varatt_external_oid8); else { Assert(false); @@ -365,7 +406,12 @@ VARATT_IS_EXTERNAL(const void *PTR) static inline bool VARATT_IS_EXTERNAL_ONDISK(const void *PTR) { - return VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_ONDISK_OID; + vartag_external tag; + + if (!VARATT_IS_EXTERNAL(PTR)) + return false; + tag = VARTAG_EXTERNAL(PTR); + return (tag == VARTAG_ONDISK_OID || tag == VARTAG_ONDISK_OID8); } /* Is varlena datum an indirect pointer? */ @@ -508,43 +554,31 @@ VARDATA_COMPRESSED_GET_COMPRESS_METHOD(const void *PTR) } /* - * Same for external Datums; but note argument is a struct - * varatt_external_oid. + * Same for external Datums, saved into an va_extinfo. */ static inline Size -VARATT_EXTERNAL_OID_GET_EXTSIZE(varatt_external_oid toast_pointer) +VARATT_EXTINFO_GET_EXTSIZE(uint32 extinfo) { - return toast_pointer.va_extinfo & VARLENA_EXTSIZE_MASK; + return extinfo & VARLENA_EXTSIZE_MASK; } static inline uint32 -VARATT_EXTERNAL_OID_GET_COMPRESS_METHOD(varatt_external_oid toast_pointer) +VARATT_EXTINFO_GET_COMPRESS_METHOD(uint32 extinfo) { - return toast_pointer.va_extinfo >> VARLENA_EXTSIZE_BITS; + return extinfo >> VARLENA_EXTSIZE_BITS; } -/* Set size and compress method of an externally-stored varlena datum */ -/* This has to remain a macro; beware multiple evaluations! */ -#define VARATT_EXTERNAL_SET_SIZE_AND_COMPRESS_METHOD(toast_pointer, len, cm) \ - do { \ - Assert((cm) == TOAST_PGLZ_COMPRESSION_ID || \ - (cm) == TOAST_LZ4_COMPRESSION_ID); \ - ((toast_pointer).va_extinfo = \ - (len) | ((uint32) (cm) << VARLENA_EXTSIZE_BITS)); \ - } while (0) - /* - * Testing whether an externally-stored value is compressed now requires - * comparing size stored in va_extinfo (the actual length of the external data) + * Testing whether an externally-stored value is compressed requires comparing + * the saved size stored in va_extinfo (the actual length of the external data) * to rawsize (the original uncompressed datum's size). The latter includes * VARHDRSZ overhead, the former doesn't. We never use compression unless it * actually saves space, so we expect either equality or less-than. */ static inline bool -VARATT_EXTERNAL_OID_IS_COMPRESSED(varatt_external_oid toast_pointer) +VARATT_EXTINFO_IS_COMPRESSED(uint32 extinfo, int32 rawsize) { - return VARATT_EXTERNAL_OID_GET_EXTSIZE(toast_pointer) < - (Size) (toast_pointer.va_rawsize - VARHDRSZ); + return VARATT_EXTINFO_GET_EXTSIZE(extinfo) < (Size) (rawsize - VARHDRSZ); } #endif diff --git a/src/backend/access/common/detoast.c b/src/backend/access/common/detoast.c index a50aea7b6b12..7e5863262b3d 100644 --- a/src/backend/access/common/detoast.c +++ b/src/backend/access/common/detoast.c @@ -225,12 +225,18 @@ detoast_attr_slice(varlena *attr, if (VARATT_IS_EXTERNAL_ONDISK(attr)) { - varatt_external_oid toast_pointer; + toast_external_data toast_ext_data; + int32 extsize; + uint32 compress_method; + bool is_compressed; - VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + toast_external_info_get(attr, &toast_ext_data); + extsize = VARATT_EXTINFO_GET_EXTSIZE(toast_ext_data.extinfo); + compress_method = VARATT_EXTINFO_GET_COMPRESS_METHOD(toast_ext_data.extinfo); + is_compressed = VARATT_EXTINFO_IS_COMPRESSED(toast_ext_data.extinfo, toast_ext_data.rawsize); /* fast path for non-compressed external datums */ - if (!VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer)) + if (!is_compressed) return toast_fetch_datum_slice(attr, sliceoffset, slicelength); /* @@ -240,7 +246,7 @@ detoast_attr_slice(varlena *attr, */ if (slicelimit >= 0) { - int32 max_size = VARATT_EXTERNAL_OID_GET_EXTSIZE(toast_pointer); + int32 max_size = extsize; /* * Determine maximum amount of compressed data needed for a prefix @@ -251,14 +257,9 @@ detoast_attr_slice(varlena *attr, * determine how much compressed data we need to be sure of being * able to decompress the required slice. */ - if (VARATT_EXTERNAL_OID_GET_COMPRESS_METHOD(toast_pointer) == - TOAST_PGLZ_COMPRESSION_ID) + if (compress_method == TOAST_PGLZ_COMPRESSION_ID) max_size = pglz_maximum_compressed_size(slicelimit, max_size); - /* - * Fetch enough compressed slices (compressed marker will get set - * automatically). - */ preslice = toast_fetch_datum_slice(attr, 0, max_size); } else @@ -344,20 +345,22 @@ toast_fetch_datum(varlena *attr) { Relation toastrel; varlena *result; - varatt_external_oid toast_pointer; + toast_external_data toast_ext_data; int32 attrsize; + Oid toastrelid; + Oid8 valueid; if (!VARATT_IS_EXTERNAL_ONDISK(attr)) elog(ERROR, "toast_fetch_datum shouldn't be called for non-ondisk datums"); - /* Must copy to access aligned fields */ - VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); - - attrsize = VARATT_EXTERNAL_OID_GET_EXTSIZE(toast_pointer); + toast_external_info_get(attr, &toast_ext_data); + attrsize = VARATT_EXTINFO_GET_EXTSIZE(toast_ext_data.extinfo); + toastrelid = toast_ext_data.toastrelid; + valueid = toast_ext_data.valueid; result = (varlena *) palloc(attrsize + VARHDRSZ); - if (VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer)) + if (VARATT_EXTINFO_IS_COMPRESSED(toast_ext_data.extinfo, toast_ext_data.rawsize)) SET_VARSIZE_COMPRESSED(result, attrsize + VARHDRSZ); else SET_VARSIZE(result, attrsize + VARHDRSZ); @@ -369,10 +372,10 @@ toast_fetch_datum(varlena *attr) /* * Open the toast relation and its indexes */ - toastrel = table_open(toast_pointer.va_toastrelid, AccessShareLock); + toastrel = table_open(toastrelid, AccessShareLock); /* Fetch all chunks */ - table_relation_fetch_toast_slice(toastrel, toast_pointer.va_valueid, + table_relation_fetch_toast_slice(toastrel, valueid, attrsize, 0, attrsize, result); /* Close toast table */ @@ -398,23 +401,27 @@ toast_fetch_datum_slice(varlena *attr, int32 sliceoffset, { Relation toastrel; varlena *result; - varatt_external_oid toast_pointer; + toast_external_data toast_ext_data; int32 attrsize; + Oid toastrelid; + Oid8 valueid; + bool is_compressed; if (!VARATT_IS_EXTERNAL_ONDISK(attr)) elog(ERROR, "toast_fetch_datum_slice shouldn't be called for non-ondisk datums"); - /* Must copy to access aligned fields */ - VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + toast_external_info_get(attr, &toast_ext_data); + attrsize = VARATT_EXTINFO_GET_EXTSIZE(toast_ext_data.extinfo); + toastrelid = toast_ext_data.toastrelid; + valueid = toast_ext_data.valueid; + is_compressed = VARATT_EXTINFO_IS_COMPRESSED(toast_ext_data.extinfo, toast_ext_data.rawsize); /* * It's nonsense to fetch slices of a compressed datum unless when it's a * prefix -- this isn't lo_* we can't return a compressed datum which is * meaningful to toast later. */ - Assert(!VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer) || 0 == sliceoffset); - - attrsize = VARATT_EXTERNAL_OID_GET_EXTSIZE(toast_pointer); + Assert(!is_compressed || 0 == sliceoffset); if (sliceoffset >= attrsize) { @@ -427,7 +434,7 @@ toast_fetch_datum_slice(varlena *attr, int32 sliceoffset, * space required by va_tcinfo, which is stored at the beginning as an * int32 value. */ - if (VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer) && slicelength > 0) + if (is_compressed && slicelength > 0) slicelength = slicelength + sizeof(int32); /* @@ -440,7 +447,7 @@ toast_fetch_datum_slice(varlena *attr, int32 sliceoffset, result = (varlena *) palloc(slicelength + VARHDRSZ); - if (VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer)) + if (is_compressed) SET_VARSIZE_COMPRESSED(result, slicelength + VARHDRSZ); else SET_VARSIZE(result, slicelength + VARHDRSZ); @@ -449,10 +456,10 @@ toast_fetch_datum_slice(varlena *attr, int32 sliceoffset, return result; /* Can save a lot of work at this point! */ /* Open the toast relation */ - toastrel = table_open(toast_pointer.va_toastrelid, AccessShareLock); + toastrel = table_open(toastrelid, AccessShareLock); /* Fetch all chunks */ - table_relation_fetch_toast_slice(toastrel, toast_pointer.va_valueid, + table_relation_fetch_toast_slice(toastrel, valueid, attrsize, sliceoffset, slicelength, result); @@ -550,10 +557,10 @@ toast_raw_datum_size(Datum value) if (VARATT_IS_EXTERNAL_ONDISK(attr)) { /* va_rawsize is the size of the original datum -- including header */ - varatt_external_oid toast_pointer; + toast_external_data toast_ext_data; - VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); - result = toast_pointer.va_rawsize; + toast_external_info_get(attr, &toast_ext_data); + result = toast_ext_data.rawsize; } else if (VARATT_IS_EXTERNAL_INDIRECT(attr)) { @@ -610,10 +617,10 @@ toast_datum_size(Datum value) * compressed or not. We do not count the size of the toast pointer * ... should we? */ - varatt_external_oid toast_pointer; + toast_external_data toast_ext_data; - VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); - result = VARATT_EXTERNAL_OID_GET_EXTSIZE(toast_pointer); + toast_external_info_get(attr, &toast_ext_data); + result = VARATT_EXTINFO_GET_EXTSIZE(toast_ext_data.extinfo); } else if (VARATT_IS_EXTERNAL_INDIRECT(attr)) { diff --git a/src/backend/access/common/toast_compression.c b/src/backend/access/common/toast_compression.c index 3b1bd6519261..09cdf880e4ff 100644 --- a/src/backend/access/common/toast_compression.c +++ b/src/backend/access/common/toast_compression.c @@ -262,12 +262,12 @@ toast_get_compression_id(varlena *attr) */ if (VARATT_IS_EXTERNAL_ONDISK(attr)) { - varatt_external_oid toast_pointer; + toast_external_data toast_ext_data; - VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + toast_external_info_get(attr, &toast_ext_data); - if (VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer)) - cmid = VARATT_EXTERNAL_OID_GET_COMPRESS_METHOD(toast_pointer); + if (VARATT_EXTINFO_IS_COMPRESSED(toast_ext_data.extinfo, toast_ext_data.rawsize)) + cmid = VARATT_EXTINFO_GET_COMPRESS_METHOD(toast_ext_data.extinfo); } else if (VARATT_IS_COMPRESSED(attr)) cmid = VARDATA_COMPRESSED_GET_COMPRESS_METHOD(attr); diff --git a/src/backend/access/common/toast_internals.c b/src/backend/access/common/toast_internals.c index f17e7b8813cb..328f98e338b9 100644 --- a/src/backend/access/common/toast_internals.c +++ b/src/backend/access/common/toast_internals.c @@ -25,7 +25,6 @@ #include "utils/fmgroids.h" #include "utils/rel.h" #include "utils/snapmgr.h" -#include "utils/lsyscache.h" static bool toastrel_valueid_exists(Relation toastrel, Oid8 valueid); static bool toastid_valueid_exists(Oid toastrelid, Oid8 valueid); @@ -104,6 +103,57 @@ toast_compress_datum(Datum value, char cmethod) } } +/* ---------- + * toast_preserve_valueid - + * + * During a table rewrite that preserves rd_toastoid, we want to preserve + * toast value IDs too. If the datum previously had an external value from + * that same toast table, return its value ID so the caller can re-use it, + * otherwise return InvalidOid8. + * + * This works for both Oid and Oid8 value IDs, as the value ID is decoded + * independently of the pointer's vartag. + * + * There is a corner case here: the table rewrite might have to copy both + * live and recently-dead versions of a row, and those versions could easily + * reference the same toast value. When we copy the second or later version + * of such a row, preserving the value ID means we select one that's already + * in the new toast table. We detect that and set *data_todo to 0 so the + * caller falls through without writing the data again. + * + * While annoying and ugly-looking, this is a good thing because it ensures + * that we wind up with only one copy of the toast value when there is only + * one copy in the old toast table. Before we detected this case, we'd have + * made multiple copies, wasting space; and what's worse, the copies + * belonging to already-deleted heap tuples would not be reclaimed by VACUUM. + * ---------- + */ +static Oid8 +toast_preserve_valueid(Relation toastrel, Oid toastoid, + varlena *oldexternal, int32 *data_todo) +{ + toast_external_data old; + + if (!OidIsValid(toastoid) || oldexternal == NULL) + return InvalidOid8; + + Assert(VARATT_IS_EXTERNAL_ONDISK(oldexternal)); + toast_external_info_get(oldexternal, &old); + + /* + * Only re-use the value ID if the old pointer came from this same toast + * table. Since that is the very same relation, its chunk_id type + * necessarily matches, so no separate vartag check is needed. + */ + if (old.toastrelid != toastoid) + return InvalidOid8; + + if (toastrel_valueid_exists(toastrel, old.valueid)) + *data_todo = 0; + + return old.valueid; +} + /* ---------- * toast_save_datum - * @@ -125,14 +175,20 @@ toast_save_datum(Relation rel, Datum value, TupleDesc toasttupDesc; CommandId mycid = GetCurrentCommandId(true); varlena *result; - varatt_external_oid toast_pointer; int32 chunk_seq = 0; char *data_p; int32 data_todo; Pointer dval = DatumGetPointer(value); int num_indexes; int validIndex; - Oid toast_typid = get_atttype(rel->rd_rel->reltoastrelid, 1); + Oid toast_typid = RelationGetToastChunkIdType(rel); + int32 max_chunk_size; + + /* Fields that will be assembled into the TOAST pointer at the end */ + int32 va_rawsize; + uint32 va_extinfo; + Oid8 va_valueid; + Oid va_toastrelid; Assert(!VARATT_IS_EXTERNAL(dval)); @@ -164,28 +220,32 @@ toast_save_datum(Relation rel, Datum value, { data_p = VARDATA_SHORT(dval); data_todo = VARSIZE_SHORT(dval) - VARHDRSZ_SHORT; - toast_pointer.va_rawsize = data_todo + VARHDRSZ; /* as if not short */ - toast_pointer.va_extinfo = data_todo; + va_rawsize = data_todo + VARHDRSZ; /* as if not short */ + va_extinfo = data_todo; } else if (VARATT_IS_COMPRESSED(dval)) { + uint32 cmid = VARDATA_COMPRESSED_GET_COMPRESS_METHOD(dval); + data_p = VARDATA(dval); data_todo = VARSIZE(dval) - VARHDRSZ; /* rawsize in a compressed datum is just the size of the payload */ - toast_pointer.va_rawsize = VARDATA_COMPRESSED_GET_EXTSIZE(dval) + VARHDRSZ; + va_rawsize = VARDATA_COMPRESSED_GET_EXTSIZE(dval) + VARHDRSZ; /* set external size and compression method */ - VARATT_EXTERNAL_SET_SIZE_AND_COMPRESS_METHOD(toast_pointer, data_todo, - VARDATA_COMPRESSED_GET_COMPRESS_METHOD(dval)); + Assert(cmid == TOAST_PGLZ_COMPRESSION_ID || + cmid == TOAST_LZ4_COMPRESSION_ID); + va_extinfo = data_todo | (cmid << VARLENA_EXTSIZE_BITS); + /* Assert that the numbers look like it's compressed */ - Assert(VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer)); + Assert(VARATT_EXTINFO_IS_COMPRESSED(va_extinfo, va_rawsize)); } else { data_p = VARDATA(dval); data_todo = VARSIZE(dval) - VARHDRSZ; - toast_pointer.va_rawsize = VARSIZE(dval); - toast_pointer.va_extinfo = data_todo; + va_rawsize = VARSIZE(dval); + va_extinfo = data_todo; } /* @@ -197,101 +257,49 @@ toast_save_datum(Relation rel, Datum value, * if we have to substitute such an OID. */ if (OidIsValid(rel->rd_toastoid)) - toast_pointer.va_toastrelid = rel->rd_toastoid; + va_toastrelid = rel->rd_toastoid; else - toast_pointer.va_toastrelid = RelationGetRelid(toastrel); + va_toastrelid = RelationGetRelid(toastrel); /* - * Choose a new value to use as the value ID for this toast value, be it - * for OID or OID8 TOAST relations. - * - * Normally we just choose an unused value within the toast table. But - * during table-rewriting operations where we are preserving an existing - * toast table OID, we want to preserve toast value IDs too. So, if - * rd_toastoid is set and we had a prior external value from that same - * toast table, re-use its value ID. If we didn't have a prior external - * value (which is a corner case, but possible if the table's attstorage - * options have been changed), we have to pick a value ID that doesn't - * conflict with either new or existing toast value IDs. If the TOAST - * table uses 8-byte value IDs, we should not really care much about - * that. + * Choose the value ID for this toast value. During a table rewrite that + * preserves rd_toastoid we re-use the prior value ID if we can (see + * toast_preserve_valueid); otherwise we pick a fresh one. For Oid value + * IDs the fresh one must not conflict with old or new toast values; for + * Oid8 the ID space is large enough that conflicts are not a concern. */ - if (!OidIsValid(rel->rd_toastoid)) + if (toast_typid == OID8OID) { - /* normal case: just choose an unused OID */ - if (toast_typid == OID8OID) - toast_pointer.va_valueid = GetNewObjectId8(); - else + va_valueid = toast_preserve_valueid(toastrel, rel->rd_toastoid, + oldexternal, &data_todo); + if (va_valueid == InvalidOid8) { - toast_pointer.va_valueid = - GetNewOidWithIndex(toastrel, - RelationGetRelid(toastidxs[validIndex]), - (AttrNumber) 1); + /* The Oid8 space is large enough that we need not check conflicts */ + va_valueid = GetNewObjectId8(); } } else { - /* rewrite case: check to see if value was in old toast table */ - toast_pointer.va_valueid = InvalidOid; - if (oldexternal != NULL) - { - varatt_external_oid old_toast_pointer; - - Assert(VARATT_IS_EXTERNAL_ONDISK(oldexternal)); - /* Must copy to access aligned fields */ - VARATT_EXTERNAL_GET_POINTER(old_toast_pointer, oldexternal); - if (old_toast_pointer.va_toastrelid == rel->rd_toastoid) - { - /* This value came from the old toast table; reuse its OID */ - toast_pointer.va_valueid = old_toast_pointer.va_valueid; - - /* - * There is a corner case here: the table rewrite might have - * to copy both live and recently-dead versions of a row, and - * those versions could easily reference the same toast value. - * When we copy the second or later version of such a row, - * reusing the OID will mean we select an OID that's already - * in the new toast table. Check for that, and if so, just - * fall through without writing the data again. - * - * While annoying and ugly-looking, this is a good thing - * because it ensures that we wind up with only one copy of - * the toast value when there is only one copy in the old - * toast table. Before we detected this case, we'd have made - * multiple copies, wasting space; and what's worse, the - * copies belonging to already-deleted heap tuples would not - * be reclaimed by VACUUM. - */ - if (toastrel_valueid_exists(toastrel, - toast_pointer.va_valueid)) - { - /* Match, so short-circuit the data storage loop below */ - data_todo = 0; - } - } - } - if (toast_pointer.va_valueid == InvalidOid) + va_valueid = toast_preserve_valueid(toastrel, rel->rd_toastoid, + oldexternal, &data_todo); + if (va_valueid == InvalidOid) { /* - * new value; must choose a value that doesn't conflict in either - * old or new toast table. + * Choose an unused OID. During a rewrite we must also avoid IDs + * that are still present in the old toast table. */ - if (toast_typid == OID8OID) - toast_pointer.va_valueid = GetNewObjectId8(); - else - { - do - { - toast_pointer.va_valueid = - GetNewOidWithIndex(toastrel, - RelationGetRelid(toastidxs[validIndex]), - (AttrNumber) 1); - } while (toastid_valueid_exists(rel->rd_toastoid, - toast_pointer.va_valueid)); - } + do + va_valueid = + GetNewOidWithIndex(toastrel, + RelationGetRelid(toastidxs[validIndex]), + (AttrNumber) 1); + while (OidIsValid(rel->rd_toastoid) && + toastid_valueid_exists(rel->rd_toastoid, va_valueid)); } } + max_chunk_size = TOAST_MAX_CHUNK_SIZE(toast_typid); + /* * Split up the item into chunks */ @@ -313,15 +321,15 @@ toast_save_datum(Relation rel, Datum value, /* * Calculate the size of this chunk */ - chunk_size = Min(TOAST_OID_MAX_CHUNK_SIZE, data_todo); + chunk_size = Min(max_chunk_size, data_todo); /* * Build a tuple and store it */ if (toast_typid == OID8OID) - t_values[0] = ObjectId8GetDatum(toast_pointer.va_valueid); + t_values[0] = ObjectId8GetDatum(va_valueid); else - t_values[0] = ObjectIdGetDatum(toast_pointer.va_valueid); + t_values[0] = ObjectIdGetDatum((Oid) va_valueid); t_values[1] = Int32GetDatum(chunk_seq++); SET_VARSIZE(&chunk_data, chunk_size + VARHDRSZ); memcpy(VARDATA(&chunk_data), data_p, chunk_size); @@ -375,11 +383,34 @@ toast_save_datum(Relation rel, Datum value, table_close(toastrel, NoLock); /* - * Create the TOAST pointer value that we'll return + * Create the TOAST pointer value that we'll return. */ - result = (varlena *) palloc(TOAST_OID_POINTER_SIZE); - SET_VARTAG_EXTERNAL(result, VARTAG_ONDISK_OID); - memcpy(VARDATA_EXTERNAL(result), &toast_pointer, sizeof(toast_pointer)); + if (toast_typid == OID8OID) + { + varatt_external_oid8 toast_pointer; + + toast_pointer.va_rawsize = va_rawsize; + toast_pointer.va_extinfo = va_extinfo; + VARATT_EXTERNAL_OID8_SET_VALUEID(&toast_pointer, va_valueid); + toast_pointer.va_toastrelid = va_toastrelid; + + result = (varlena *) palloc(TOAST_OID8_POINTER_SIZE); + SET_VARTAG_EXTERNAL(result, VARTAG_ONDISK_OID8); + memcpy(VARDATA_EXTERNAL(result), &toast_pointer, sizeof(toast_pointer)); + } + else + { + varatt_external_oid toast_pointer; + + toast_pointer.va_rawsize = va_rawsize; + toast_pointer.va_extinfo = va_extinfo; + toast_pointer.va_valueid = (Oid) va_valueid; + toast_pointer.va_toastrelid = va_toastrelid; + + result = (varlena *) palloc(TOAST_OID_POINTER_SIZE); + SET_VARTAG_EXTERNAL(result, VARTAG_ONDISK_OID); + memcpy(VARDATA_EXTERNAL(result), &toast_pointer, sizeof(toast_pointer)); + } return PointerGetDatum(result); } @@ -415,7 +446,7 @@ void toast_delete_datum(Relation rel, Datum value, bool is_speculative) { varlena *attr = (varlena *) DatumGetPointer(value); - varatt_external_oid toast_pointer; + toast_external_data toast_ext_data; Relation toastrel; Relation *toastidxs; ScanKeyData toastkey; @@ -427,26 +458,26 @@ toast_delete_datum(Relation rel, Datum value, bool is_speculative) if (!VARATT_IS_EXTERNAL_ONDISK(attr)) return; - /* Must copy to access aligned fields */ - VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + /* + * Decode the pointer to get the toast relation OID and value ID. The + * vartag tells us everything we need - no TOAST table schema lookup + * required. + */ + toast_external_info_get(attr, &toast_ext_data); /* * Open the toast relation and its indexes */ - toastrel = table_open(toast_pointer.va_toastrelid, RowExclusiveLock); - - /* Fetch valid relation used for process */ + toastrel = table_open(toast_ext_data.toastrelid, RowExclusiveLock); validIndex = toast_open_indexes(toastrel, RowExclusiveLock, &toastidxs, &num_indexes); - /* - * Setup a scan key to find chunks with matching va_valueid - */ + /* Set up a scan key to find chunks with matching value ID */ toast_valueid_scankey_init(&toastkey, (AttrNumber) 1, TupleDescAttr(toastrel->rd_att, 0)->atttypid, - toast_pointer.va_valueid); + toast_ext_data.valueid); /* * Find all the chunks. (We don't actually care whether we see them in diff --git a/src/backend/access/heap/heaptoast.c b/src/backend/access/heap/heaptoast.c index b9c89593ecf4..a3d5cdecb3c1 100644 --- a/src/backend/access/heap/heaptoast.c +++ b/src/backend/access/heap/heaptoast.c @@ -649,7 +649,7 @@ heap_fetch_toast_slice(Relation toastrel, Oid8 valueid, int32 attrsize, &num_indexes); toast_typid = TupleDescAttr(toastrel->rd_att, 0)->atttypid; - max_chunk_size = TOAST_OID_MAX_CHUNK_SIZE; + max_chunk_size = TOAST_MAX_CHUNK_SIZE(toast_typid); totalchunks = ((attrsize - 1) / max_chunk_size) + 1; startchunk = sliceoffset / max_chunk_size; diff --git a/src/backend/access/table/toast_helper.c b/src/backend/access/table/toast_helper.c index 6f2d691cc681..2613d9dc095b 100644 --- a/src/backend/access/table/toast_helper.c +++ b/src/backend/access/table/toast_helper.c @@ -69,12 +69,16 @@ toast_tuple_init(ToastTupleContext *ttc) /* * If the old value is stored on disk, check if it has changed so * we have to delete it later. + * + * Note that TOAST pointers could have different vartags, for oid + * or oid8, and these can have different sizes. */ if (att->attlen == -1 && !ttc->ttc_oldisnull[i] && VARATT_IS_EXTERNAL_ONDISK(old_value)) { if (ttc->ttc_isnull[i] || !VARATT_IS_EXTERNAL_ONDISK(new_value) || + VARTAG_EXTERNAL(old_value) != VARTAG_EXTERNAL(new_value) || memcmp(old_value, new_value, VARSIZE_EXTERNAL(old_value)) != 0) { @@ -171,8 +175,9 @@ toast_tuple_init(ToastTupleContext *ttc) * The column must have attstorage EXTERNAL or EXTENDED if check_main is * false, and must have attstorage MAIN if check_main is true. * - * The column must have a minimum size of MAXALIGN(TOAST_OID_POINTER_SIZE); - * if not, no benefit is to be expected by compressing it. + * The column must be larger than the TOAST pointer that would replace it; + * if not, no benefit is to be expected by compressing it. Note that this + * choice depends on the TOAST value type, oid or oid8. * * The return value is the index of the biggest suitable column, or * -1 if there is none. @@ -184,10 +189,21 @@ toast_tuple_find_biggest_attribute(ToastTupleContext *ttc, TupleDesc tupleDesc = ttc->ttc_rel->rd_att; int numAttrs = tupleDesc->natts; int biggest_attno = -1; - int32 biggest_size = MAXALIGN(TOAST_OID_POINTER_SIZE); + int32 biggest_size; int32 skip_colflags = TOASTCOL_IGNORE; int i; + /* + * Size of the TOAST pointer this relation would use. A relation without + * a TOAST table cannot have any of its attributes moved out-of-line, but + * it can still have some of them compressed. Fall back to the oid size + * in that case. + */ + if (RelationGetToastChunkIdType(ttc->ttc_rel) == OID8OID) + biggest_size = MAXALIGN(TOAST_OID8_POINTER_SIZE); + else + biggest_size = MAXALIGN(TOAST_OID_POINTER_SIZE); + if (for_compression) skip_colflags |= TOASTCOL_INCOMPRESSIBLE; diff --git a/src/backend/replication/logical/reorderbuffer.c b/src/backend/replication/logical/reorderbuffer.c index dd4b6e8f7f45..69a479cd31da 100644 --- a/src/backend/replication/logical/reorderbuffer.c +++ b/src/backend/replication/logical/reorderbuffer.c @@ -5175,7 +5175,7 @@ ReorderBufferToastReplace(ReorderBuffer *rb, ReorderBufferTXN *txn, varlena *varlena_pointer; /* va_rawsize is the size of the original datum -- including header */ - varatt_external_oid toast_pointer; + toast_external_data toast_ext_data; varatt_indirect redirect_pointer; varlena *new_datum = NULL; varlena *reconstructed; @@ -5198,11 +5198,11 @@ ReorderBufferToastReplace(ReorderBuffer *rb, ReorderBufferTXN *txn, varlena_pointer = (varlena *) DatumGetPointer(attrs[natt]); /* no need to do anything if the tuple isn't external */ - if (!VARATT_IS_EXTERNAL(varlena_pointer)) + if (!VARATT_IS_EXTERNAL_ONDISK(varlena_pointer)) continue; - VARATT_EXTERNAL_GET_POINTER(toast_pointer, varlena_pointer); - toast_valueid = toast_pointer.va_valueid; + toast_external_info_get(varlena_pointer, &toast_ext_data); + toast_valueid = toast_ext_data.valueid; /* * Check whether the toast tuple changed, replace if so. @@ -5220,7 +5220,7 @@ ReorderBufferToastReplace(ReorderBuffer *rb, ReorderBufferTXN *txn, free[natt] = true; - reconstructed = palloc0(toast_pointer.va_rawsize); + reconstructed = palloc0(toast_ext_data.rawsize); ent->reconstructed = reconstructed; @@ -5245,10 +5245,11 @@ ReorderBufferToastReplace(ReorderBuffer *rb, ReorderBufferTXN *txn, VARSIZE(chunk) - VARHDRSZ); data_done += VARSIZE(chunk) - VARHDRSZ; } - Assert(data_done == VARATT_EXTERNAL_OID_GET_EXTSIZE(toast_pointer)); + + Assert(data_done == VARATT_EXTINFO_GET_EXTSIZE(toast_ext_data.extinfo)); /* make sure its marked as compressed or not */ - if (VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer)) + if (VARATT_EXTINFO_IS_COMPRESSED(toast_ext_data.extinfo, toast_ext_data.rawsize)) SET_VARSIZE_COMPRESSED(reconstructed, data_done + VARHDRSZ); else SET_VARSIZE(reconstructed, data_done + VARHDRSZ); diff --git a/src/backend/utils/adt/varlena.c b/src/backend/utils/adt/varlena.c index 4533dac11a6d..f8c16bd4c871 100644 --- a/src/backend/utils/adt/varlena.c +++ b/src/backend/utils/adt/varlena.c @@ -4261,7 +4261,8 @@ pg_column_toast_chunk_id(PG_FUNCTION_ARGS) { int typlen; varlena *attr; - varatt_external_oid toast_pointer; + toast_external_data toast_ext_data; + Oid8 result; /* On first call, get the input type's typlen, and save at *fn_extra */ if (fcinfo->flinfo->fn_extra == NULL) @@ -4288,9 +4289,10 @@ pg_column_toast_chunk_id(PG_FUNCTION_ARGS) if (!VARATT_IS_EXTERNAL_ONDISK(attr)) PG_RETURN_NULL(); - VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + toast_external_info_get(attr, &toast_ext_data); + result = toast_ext_data.valueid; - PG_RETURN_OID8(toast_pointer.va_valueid); + PG_RETURN_OID8(result); } /* diff --git a/src/backend/utils/cache/relcache.c b/src/backend/utils/cache/relcache.c index f475d7039772..65dfdbda100a 100644 --- a/src/backend/utils/cache/relcache.c +++ b/src/backend/utils/cache/relcache.c @@ -1205,6 +1205,9 @@ retry: relation->rd_fkeylist = NIL; relation->rd_fkeyvalid = false; + /* TOAST type data is not loaded till asked for */ + relation->rd_toastchunkidtype = InvalidOid; + /* partitioning data is not loaded till asked for */ relation->rd_partkey = NULL; relation->rd_partkeycxt = NULL; @@ -5093,6 +5096,39 @@ RelationGetReplicaIndex(Relation relation) return relation->rd_replidindex; } +/* + * RelationGetToastChunkIdType -- get the type of the relation's TOAST + * table "chunk_id" column + * + * Returns OIDOID or OID8OID, or InvalidOid if the relation has no TOAST + * table. + */ +Oid +RelationGetToastChunkIdType(Relation relation) +{ + Oid toastrelid = relation->rd_rel->reltoastrelid; + Oid typid; + + /* Quick exit if we already computed the value */ + if (OidIsValid(relation->rd_toastchunkidtype)) + return relation->rd_toastchunkidtype; + + /* Nothing to report without a TOAST table */ + if (!OidIsValid(toastrelid)) + return InvalidOid; + + typid = get_atttype(toastrelid, 1); + if (!OidIsValid(typid)) + elog(ERROR, "cache lookup failed for TOAST relation %u", + toastrelid); + if (typid != OIDOID && typid != OID8OID) + elog(ERROR, "unexpected type %u for chunk_id in TOAST relation %u", + typid, toastrelid); + + relation->rd_toastchunkidtype = typid; + return typid; +} + /* * RelationGetIndexExpressions -- get the index expressions for an index * @@ -6530,6 +6566,7 @@ load_relcache_init_file(bool shared) rel->rd_firstRelfilelocatorSubid = InvalidSubTransactionId; rel->rd_droppedSubid = InvalidSubTransactionId; rel->rd_amcache = NULL; + rel->rd_toastchunkidtype = InvalidOid; rel->pgstat_info = NULL; /* diff --git a/doc/src/sgml/storage.sgml b/doc/src/sgml/storage.sgml index 21bd67c11ea0..83de016eaa58 100644 --- a/doc/src/sgml/storage.sgml +++ b/doc/src/sgml/storage.sgml @@ -417,8 +417,10 @@ described in more detail below. Out-of-line values are divided (after compression if used) into chunks of at -most TOAST_OID_MAX_CHUNK_SIZE bytes (by default this value is chosen -so that four chunk rows will fit on a page, making it about 2000 bytes). +most TOAST_OID_MAX_CHUNK_SIZE or +TOAST_OID8_MAX_CHUNK_SIZE bytes depending on the +chunk_id type (by default these values are chosen +so that four chunk rows will fit on a page, making them about 2000 bytes). Each chunk is stored as a separate row in the TOAST table belonging to the owning table. Every TOAST table has the columns @@ -434,8 +436,10 @@ retrieval of the values. A pointer datum representing an out-of-line on-disk logical datum size (original uncompressed data length), physical stored size (different if compression was applied), and the compression method used, if any. Allowing for the varlena header bytes, -the total size of an on-disk TOAST pointer datum is therefore 18 -bytes regardless of the actual size of the represented value. +the total size of an on-disk TOAST pointer datum is 18 +bytes when using an OID as chunk_id, or 22 bytes +when using an OID8 as chunk_id, regardless of the +actual size of the represented value. diff --git a/contrib/amcheck/verify_heapam.c b/contrib/amcheck/verify_heapam.c index c89fab8764ac..edecbd792b0c 100644 --- a/contrib/amcheck/verify_heapam.c +++ b/contrib/amcheck/verify_heapam.c @@ -74,7 +74,9 @@ typedef enum SkipPages */ typedef struct ToastedAttribute { - varatt_external_oid toast_pointer; + vartag_external tag; /* VARTAG_ONDISK_OID or VARTAG_ONDISK_OID8 */ + Oid8 va_valueid; /* value ID (works for both Oid and Oid8) */ + uint32 va_extinfo; /* external size and compression method */ BlockNumber blkno; /* block in main table */ OffsetNumber offnum; /* offset in main table */ AttrNumber attnum; /* attribute in main table */ @@ -1565,9 +1567,12 @@ check_toast_tuple(HeapTuple toasttup, HeapCheckContext *ctx, int32 max_chunk_size; Oid8 toast_valueid; - max_chunk_size = TOAST_OID_MAX_CHUNK_SIZE; + toast_valueid = ta->va_valueid; + + max_chunk_size = ta->tag == VARTAG_ONDISK_OID8 + ? TOAST_OID8_MAX_CHUNK_SIZE + : TOAST_OID_MAX_CHUNK_SIZE; last_chunk_seq = (extsize - 1) / max_chunk_size; - toast_valueid = ta->toast_pointer.va_valueid; /* Sanity-check the sequence number. */ chunk_seq = DatumGetInt32(fastgetattr(toasttup, 2, @@ -1671,8 +1676,11 @@ check_tuple_attribute(HeapCheckContext *ctx) char *tp; /* pointer to the tuple data */ uint16 infomask; Oid8 toast_pointer_valueid; + int32 va_rawsize; + uint32 va_extinfo; CompactAttribute *thisatt; - varatt_external_oid toast_pointer; + vartag_external va_tag_value; + toast_external_data toast_ext_data; infomask = ctx->tuphdr->t_infomask; thisatt = TupleDescCompactAttr(RelationGetDescr(ctx->rel), ctx->attnum); @@ -1731,7 +1739,7 @@ check_tuple_attribute(HeapCheckContext *ctx) { uint8 va_tag = VARTAG_EXTERNAL(tp + ctx->offset); - if (va_tag != VARTAG_ONDISK_OID) + if (va_tag != VARTAG_ONDISK_OID && va_tag != VARTAG_ONDISK_OID8) { report_corruption(ctx, psprintf("toasted attribute has unexpected TOAST tag %u", @@ -1775,27 +1783,28 @@ check_tuple_attribute(HeapCheckContext *ctx) /* It is external, and we're looking at a page on disk */ - /* - * Must copy attr into toast_pointer for alignment considerations - */ - VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); - toast_pointer_valueid = toast_pointer.va_valueid; + /* Must copy attr into a decoded pointer for alignment considerations */ + toast_external_info_get(attr, &toast_ext_data); + va_tag_value = toast_ext_data.tag; + toast_pointer_valueid = toast_ext_data.valueid; + va_rawsize = toast_ext_data.rawsize; + va_extinfo = toast_ext_data.extinfo; /* Toasted attributes too large to be untoasted should never be stored */ - if (toast_pointer.va_rawsize > VARLENA_SIZE_LIMIT) + if (va_rawsize > VARLENA_SIZE_LIMIT) report_corruption(ctx, psprintf("toast value " OID8_FORMAT " rawsize %d exceeds limit %d", toast_pointer_valueid, - toast_pointer.va_rawsize, + va_rawsize, VARLENA_SIZE_LIMIT)); - if (VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer)) + if (VARATT_EXTINFO_IS_COMPRESSED(toast_ext_data.extinfo, toast_ext_data.rawsize)) { ToastCompressionId cmid; bool valid = false; /* Compressed attributes should have a valid compression method */ - cmid = TOAST_COMPRESS_METHOD(&toast_pointer); + cmid = VARATT_EXTINFO_GET_COMPRESS_METHOD(toast_ext_data.extinfo); switch (cmid) { /* List of all valid compression method IDs */ @@ -1849,7 +1858,10 @@ check_tuple_attribute(HeapCheckContext *ctx) ta = palloc0_object(ToastedAttribute); - VARATT_EXTERNAL_GET_POINTER(ta->toast_pointer, attr); + /* The pointer has already been decoded above, just reuse it */ + ta->tag = va_tag_value; + ta->va_valueid = toast_pointer_valueid; + ta->va_extinfo = va_extinfo; ta->blkno = ctx->blkno; ta->offnum = ctx->offnum; ta->attnum = ctx->attnum; @@ -1879,17 +1891,18 @@ check_toasted_attribute(HeapCheckContext *ctx, ToastedAttribute *ta) Oid8 toast_valueid; Oid toast_typid; - toast_typid = TupleDescAttr(ctx->toast_rel->rd_att, 0)->atttypid; - max_chunk_size = TOAST_OID_MAX_CHUNK_SIZE; - - extsize = VARATT_EXTERNAL_OID_GET_EXTSIZE(ta->toast_pointer); - last_chunk_seq = (extsize - 1) / max_chunk_size; + toast_valueid = ta->va_valueid; + extsize = VARATT_EXTINFO_GET_EXTSIZE(ta->va_extinfo); + toast_typid = (ta->tag == VARTAG_ONDISK_OID8) ? OID8OID : OIDOID; /* * Setup a scan key to find chunks in toast table with matching va_valueid */ + max_chunk_size = TOAST_MAX_CHUNK_SIZE(toast_typid); toast_valueid_scankey_init(&toastkey, (AttrNumber) 1, toast_typid, - ta->toast_pointer.va_valueid); + toast_valueid); + + last_chunk_seq = (extsize - 1) / max_chunk_size; /* * Check if any chunks for this toasted object exist in the toast table, @@ -1909,8 +1922,6 @@ check_toasted_attribute(HeapCheckContext *ctx, ToastedAttribute *ta) } systable_endscan_ordered(toastscan); - toast_valueid = ta->toast_pointer.va_valueid; - if (!found_toasttup) report_toast_corruption(ctx, ta, psprintf("toast value " OID8_FORMAT " not found in toast table", diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index de21cea65f96..9985c8b21e31 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -4358,6 +4358,7 @@ timeout_params timerCA tlist_vinfo toast_compress_header +toast_external_data tokenize_error_callback_arg transferMode transfer_thread_arg @@ -4414,6 +4415,7 @@ vacuumingOptions validate_string_relopt varatt_expanded varatt_external_oid +varatt_external_oid8 varatt_indirect varattrib_1b varattrib_1b_e -- 2.55.0