From a39c74131923f0e62e8339eff31f46adf439aa61 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Tue, 8 Sep 2026 11:48:29 +0900 Subject: [PATCH v15 10/11] Add support for TOAST pointers as oid8 This introduces a new varlena external pointer structure, varatt_external_oid8, that carries a 64-bit value ID (Oid8) for TOAST tables using oid8 as their chunk_id type. XXX: Catalog version bump required. --- src/include/access/detoast.h | 3 + src/include/access/heaptoast.h | 16 +- src/include/utils/rel.h | 2 +- src/include/varatt.h | 93 ++++- src/backend/access/common/detoast.c | 151 ++++++-- src/backend/access/common/toast_compression.c | 11 +- src/backend/access/common/toast_internals.c | 353 ++++++++++++------ src/backend/access/heap/heaptoast.c | 17 +- src/backend/access/table/toast_helper.c | 4 + src/backend/catalog/toasting.c | 19 +- .../replication/logical/reorderbuffer.c | 45 ++- src/backend/utils/adt/varlena.c | 19 +- doc/src/sgml/ref/create_table.sgml | 25 +- doc/src/sgml/storage.sgml | 10 +- contrib/amcheck/verify_heapam.c | 90 +++-- 15 files changed, 631 insertions(+), 227 deletions(-) diff --git a/src/include/access/detoast.h b/src/include/access/detoast.h index 2b1428b1e81c..f065ea034e98 100644 --- a/src/include/access/detoast.h +++ b/src/include/access/detoast.h @@ -30,6 +30,9 @@ do { \ /* Size of an EXTERNAL datum that contains a standard TOAST pointer */ #define TOAST_OID_POINTER_SIZE (VARHDRSZ_EXTERNAL + sizeof(varatt_external_oid)) +/* Size of an EXTERNAL datum that contains an Oid8 TOAST pointer */ +#define TOAST_OID8_POINTER_SIZE (VARHDRSZ_EXTERNAL + sizeof(varatt_external_oid8)) + /* Size of an EXTERNAL datum that contains an indirection pointer */ #define INDIRECT_POINTER_SIZE (VARHDRSZ_EXTERNAL + sizeof(varatt_indirect)) diff --git a/src/include/access/heaptoast.h b/src/include/access/heaptoast.h index aaae6e0fef69..b7ef69a8f56b 100644 --- a/src/include/access/heaptoast.h +++ b/src/include/access/heaptoast.h @@ -69,13 +69,14 @@ /* * When we store an oversize datum externally, we divide it into chunks - * containing at most TOAST_OID_MAX_CHUNK_SIZE data bytes. This number *must* - * be small enough that the completed toast-table tuple (including the - * ID and sequence fields and all overhead) will fit on a page. + * containing at most TOAST_OID_MAX_CHUNK_SIZE or TOAST_OID8_MAX_CHUNK_SIZE + * data bytes, depending on the chunk_id type of the TOAST table. These + * numbers *must* be small enough that the completed toast-table tuple + * (including the ID and sequence fields and all overhead) will fit on a page. * The coding here sets the size on the theory that we want to fit * EXTERN_TUPLES_PER_PAGE tuples of maximum size onto a page. * - * NB: Changing TOAST_OID_MAX_CHUNK_SIZE requires an initdb. + * NB: Changing these values requires an initdb. */ #define EXTERN_TUPLES_PER_PAGE 4 /* tweak only this */ @@ -88,6 +89,13 @@ sizeof(int32) - \ VARHDRSZ) +#define TOAST_OID8_MAX_CHUNK_SIZE \ + (EXTERN_TUPLE_MAX_SIZE - \ + MAXALIGN(SizeofHeapTupleHeader) - \ + sizeof(Oid8) - \ + sizeof(int32) - \ + VARHDRSZ) + /* ---------- * heap_toast_insert_or_update - * diff --git a/src/include/utils/rel.h b/src/include/utils/rel.h index b7da03446a94..8bec73d55794 100644 --- a/src/include/utils/rel.h +++ b/src/include/utils/rel.h @@ -386,7 +386,7 @@ typedef struct StdRdOptions */ #define RelationGetToastValueType(relation, defaulttarg) \ ((relation)->rd_options ? \ - ((StdRdOptions *) (relation)->rd_options)->toast_value_type : defaulttarg) + ((StdRdOptions *) (relation)->rd_options)->toast_value_type : (defaulttarg)) /* * RelationGetFillFactor diff --git a/src/include/varatt.h b/src/include/varatt.h index 33979bbaaad1..1349a54371db 100644 --- a/src/include/varatt.h +++ b/src/include/varatt.h @@ -38,6 +38,44 @@ typedef struct varatt_external_oid Oid va_toastrelid; /* RelID of TOAST table containing it */ } varatt_external_oid; +/* + * varatt_external_oid8 is a "TOAST pointer" for TOAST tables that use + * Oid8 (64-bit) as their chunk_id type. Same layout as varatt_external_oid + * except for the value ID, which is 8 bytes wide. The value ID is split + * into two uint32 to force alignment. + * + * This struct must not contain any padding, because we sometimes compare + * these pointers using memcmp. + */ +typedef struct varatt_external_oid8 +{ + int32 va_rawsize; /* Original data size (includes header) */ + uint32 va_extinfo; /* External saved size (without header) and + * compression method */ + uint32 va_valueid_lo; /* Low 32 bits of value ID */ + uint32 va_valueid_hi; /* High 32 bits of value ID */ + Oid va_toastrelid; /* RelID of TOAST table containing it */ +} varatt_external_oid8; + +StaticAssertDecl((sizeof(int32) + 3 * sizeof(uint32) + sizeof(Oid)) == + sizeof(varatt_external_oid8), + "varatt_external_oid8 should have no padding"); + +/* Get/set the 64-bit value ID from the lo/hi halves */ +static inline Oid8 +VARATT_EXTERNAL_OID8_GET_VALUEID(varatt_external_oid8 toast_pointer) +{ + return ((Oid8) toast_pointer.va_valueid_lo) | + (((Oid8) toast_pointer.va_valueid_hi) << 32); +} + +static inline void +VARATT_EXTERNAL_OID8_SET_VALUEID(varatt_external_oid8 *toast_pointer, Oid8 id) +{ + toast_pointer->va_valueid_lo = (uint32) id; + toast_pointer->va_valueid_hi = (uint32) (id >> 32); +} + /* * These macros define the "saved size" portion of va_extinfo. Its remaining * two high-order bits identify the compression method. @@ -87,6 +125,7 @@ typedef enum vartag_external VARTAG_INDIRECT = 1, VARTAG_EXPANDED_RO = 2, VARTAG_EXPANDED_RW = 3, + VARTAG_ONDISK_OID8 = 4, VARTAG_ONDISK_OID = 18 } vartag_external; @@ -108,6 +147,8 @@ VARTAG_SIZE(vartag_external tag) return sizeof(varatt_expanded); else if (tag == VARTAG_ONDISK_OID) return sizeof(varatt_external_oid); + else if (tag == VARTAG_ONDISK_OID8) + return sizeof(varatt_external_oid8); else { Assert(false); @@ -360,10 +401,29 @@ VARATT_IS_EXTERNAL(const void *PTR) /* Is varlena datum a pointer to on-disk toasted data? */ static inline bool VARATT_IS_EXTERNAL_ONDISK(const void *PTR) +{ + vartag_external tag; + + if (!VARATT_IS_EXTERNAL(PTR)) + return false; + tag = VARTAG_EXTERNAL(PTR); + return (tag == VARTAG_ONDISK_OID || tag == VARTAG_ONDISK_OID8); +} + +/* Is varlena datum a pointer to on-disk toasted data with 4-byte value ID? */ +static inline bool +VARATT_IS_EXTERNAL_ONDISK_OID(const void *PTR) { return VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_ONDISK_OID; } +/* Is varlena datum a pointer to on-disk toasted data with 8-byte value ID? */ +static inline bool +VARATT_IS_EXTERNAL_ONDISK_OID8(const void *PTR) +{ + return VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_ONDISK_OID8; +} + /* Is varlena datum an indirect pointer? */ static inline bool VARATT_IS_EXTERNAL_INDIRECT(const void *PTR) @@ -519,16 +579,6 @@ VARATT_EXTERNAL_OID_GET_COMPRESS_METHOD(varatt_external_oid toast_pointer) return toast_pointer.va_extinfo >> VARLENA_EXTSIZE_BITS; } -/* Set size and compress method of an externally-stored varlena datum */ -/* This has to remain a macro; beware multiple evaluations! */ -#define VARATT_EXTERNAL_SET_SIZE_AND_COMPRESS_METHOD(toast_pointer, len, cm) \ - do { \ - Assert((cm) == TOAST_PGLZ_COMPRESSION_ID || \ - (cm) == TOAST_LZ4_COMPRESSION_ID); \ - ((toast_pointer).va_extinfo = \ - (len) | ((uint32) (cm) << VARLENA_EXTSIZE_BITS)); \ - } while (0) - /* * Testing whether an externally-stored value is compressed now requires * comparing size stored in va_extinfo (the actual length of the external data) @@ -543,4 +593,27 @@ VARATT_EXTERNAL_OID_IS_COMPRESSED(varatt_external_oid toast_pointer) (Size) (toast_pointer.va_rawsize - VARHDRSZ); } +/* + * Same for external Datums; but note argument is a struct + * varatt_external_oid8. + */ +static inline Size +VARATT_EXTERNAL_OID8_GET_EXTSIZE(varatt_external_oid8 toast_pointer) +{ + return toast_pointer.va_extinfo & VARLENA_EXTSIZE_MASK; +} + +static inline uint32 +VARATT_EXTERNAL_OID8_GET_COMPRESS_METHOD(varatt_external_oid8 toast_pointer) +{ + return toast_pointer.va_extinfo >> VARLENA_EXTSIZE_BITS; +} + +static inline bool +VARATT_EXTERNAL_OID8_IS_COMPRESSED(varatt_external_oid8 toast_pointer) +{ + return VARATT_EXTERNAL_OID8_GET_EXTSIZE(toast_pointer) < + (Size) (toast_pointer.va_rawsize - VARHDRSZ); +} + #endif diff --git a/src/backend/access/common/detoast.c b/src/backend/access/common/detoast.c index a50aea7b6b12..393135f1b9f3 100644 --- a/src/backend/access/common/detoast.c +++ b/src/backend/access/common/detoast.c @@ -225,12 +225,32 @@ detoast_attr_slice(varlena *attr, if (VARATT_IS_EXTERNAL_ONDISK(attr)) { - varatt_external_oid toast_pointer; + int32 extsize; + uint32 compress_method; + bool is_compressed; - VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + /* Extract extsize and compression info from the appropriate pointer */ + if (VARTAG_EXTERNAL(attr) == VARTAG_ONDISK_OID8) + { + varatt_external_oid8 toast_pointer; + + VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + extsize = VARATT_EXTERNAL_OID8_GET_EXTSIZE(toast_pointer); + compress_method = VARATT_EXTERNAL_OID8_GET_COMPRESS_METHOD(toast_pointer); + is_compressed = VARATT_EXTERNAL_OID8_IS_COMPRESSED(toast_pointer); + } + else + { + varatt_external_oid toast_pointer; + + VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + extsize = VARATT_EXTERNAL_OID_GET_EXTSIZE(toast_pointer); + compress_method = VARATT_EXTERNAL_OID_GET_COMPRESS_METHOD(toast_pointer); + is_compressed = VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer); + } /* fast path for non-compressed external datums */ - if (!VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer)) + if (!is_compressed) return toast_fetch_datum_slice(attr, sliceoffset, slicelength); /* @@ -240,7 +260,7 @@ detoast_attr_slice(varlena *attr, */ if (slicelimit >= 0) { - int32 max_size = VARATT_EXTERNAL_OID_GET_EXTSIZE(toast_pointer); + int32 max_size = extsize; /* * Determine maximum amount of compressed data needed for a prefix @@ -251,14 +271,9 @@ detoast_attr_slice(varlena *attr, * determine how much compressed data we need to be sure of being * able to decompress the required slice. */ - if (VARATT_EXTERNAL_OID_GET_COMPRESS_METHOD(toast_pointer) == - TOAST_PGLZ_COMPRESSION_ID) + if (compress_method == TOAST_PGLZ_COMPRESSION_ID) max_size = pglz_maximum_compressed_size(slicelimit, max_size); - /* - * Fetch enough compressed slices (compressed marker will get set - * automatically). - */ preslice = toast_fetch_datum_slice(attr, 0, max_size); } else @@ -344,23 +359,47 @@ toast_fetch_datum(varlena *attr) { Relation toastrel; varlena *result; - varatt_external_oid toast_pointer; int32 attrsize; + Oid toastrelid; + Oid8 valueid; if (!VARATT_IS_EXTERNAL_ONDISK(attr)) elog(ERROR, "toast_fetch_datum shouldn't be called for non-ondisk datums"); - /* Must copy to access aligned fields */ - VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + if (VARTAG_EXTERNAL(attr) == VARTAG_ONDISK_OID8) + { + varatt_external_oid8 toast_pointer; - attrsize = VARATT_EXTERNAL_OID_GET_EXTSIZE(toast_pointer); + /* Must copy to access aligned fields */ + VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + attrsize = VARATT_EXTERNAL_OID8_GET_EXTSIZE(toast_pointer); + toastrelid = toast_pointer.va_toastrelid; + valueid = VARATT_EXTERNAL_OID8_GET_VALUEID(toast_pointer); - result = (varlena *) palloc(attrsize + VARHDRSZ); + result = (varlena *) palloc(attrsize + VARHDRSZ); - if (VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer)) - SET_VARSIZE_COMPRESSED(result, attrsize + VARHDRSZ); + if (VARATT_EXTERNAL_OID8_IS_COMPRESSED(toast_pointer)) + SET_VARSIZE_COMPRESSED(result, attrsize + VARHDRSZ); + else + SET_VARSIZE(result, attrsize + VARHDRSZ); + } else - SET_VARSIZE(result, attrsize + VARHDRSZ); + { + varatt_external_oid toast_pointer; + + /* Must copy to access aligned fields */ + VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + attrsize = VARATT_EXTERNAL_OID_GET_EXTSIZE(toast_pointer); + toastrelid = toast_pointer.va_toastrelid; + valueid = toast_pointer.va_valueid; + + result = (varlena *) palloc(attrsize + VARHDRSZ); + + if (VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer)) + SET_VARSIZE_COMPRESSED(result, attrsize + VARHDRSZ); + else + SET_VARSIZE(result, attrsize + VARHDRSZ); + } if (attrsize == 0) return result; /* Probably shouldn't happen, but just in @@ -369,10 +408,10 @@ toast_fetch_datum(varlena *attr) /* * Open the toast relation and its indexes */ - toastrel = table_open(toast_pointer.va_toastrelid, AccessShareLock); + toastrel = table_open(toastrelid, AccessShareLock); /* Fetch all chunks */ - table_relation_fetch_toast_slice(toastrel, toast_pointer.va_valueid, + table_relation_fetch_toast_slice(toastrel, valueid, attrsize, 0, attrsize, result); /* Close toast table */ @@ -398,23 +437,43 @@ toast_fetch_datum_slice(varlena *attr, int32 sliceoffset, { Relation toastrel; varlena *result; - varatt_external_oid toast_pointer; int32 attrsize; + Oid toastrelid; + Oid8 valueid; + bool is_compressed; if (!VARATT_IS_EXTERNAL_ONDISK(attr)) elog(ERROR, "toast_fetch_datum_slice shouldn't be called for non-ondisk datums"); - /* Must copy to access aligned fields */ - VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + if (VARTAG_EXTERNAL(attr) == VARTAG_ONDISK_OID8) + { + varatt_external_oid8 toast_pointer; + + /* Must copy to access aligned fields */ + VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + attrsize = VARATT_EXTERNAL_OID8_GET_EXTSIZE(toast_pointer); + toastrelid = toast_pointer.va_toastrelid; + valueid = VARATT_EXTERNAL_OID8_GET_VALUEID(toast_pointer); + is_compressed = VARATT_EXTERNAL_OID8_IS_COMPRESSED(toast_pointer); + } + else + { + varatt_external_oid toast_pointer; + + /* Must copy to access aligned fields */ + VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + attrsize = VARATT_EXTERNAL_OID_GET_EXTSIZE(toast_pointer); + toastrelid = toast_pointer.va_toastrelid; + valueid = toast_pointer.va_valueid; + is_compressed = VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer); + } /* * It's nonsense to fetch slices of a compressed datum unless when it's a * prefix -- this isn't lo_* we can't return a compressed datum which is * meaningful to toast later. */ - Assert(!VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer) || 0 == sliceoffset); - - attrsize = VARATT_EXTERNAL_OID_GET_EXTSIZE(toast_pointer); + Assert(!is_compressed || 0 == sliceoffset); if (sliceoffset >= attrsize) { @@ -427,7 +486,7 @@ toast_fetch_datum_slice(varlena *attr, int32 sliceoffset, * space required by va_tcinfo, which is stored at the beginning as an * int32 value. */ - if (VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer) && slicelength > 0) + if (is_compressed && slicelength > 0) slicelength = slicelength + sizeof(int32); /* @@ -440,7 +499,7 @@ toast_fetch_datum_slice(varlena *attr, int32 sliceoffset, result = (varlena *) palloc(slicelength + VARHDRSZ); - if (VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer)) + if (is_compressed) SET_VARSIZE_COMPRESSED(result, slicelength + VARHDRSZ); else SET_VARSIZE(result, slicelength + VARHDRSZ); @@ -449,10 +508,10 @@ toast_fetch_datum_slice(varlena *attr, int32 sliceoffset, return result; /* Can save a lot of work at this point! */ /* Open the toast relation */ - toastrel = table_open(toast_pointer.va_toastrelid, AccessShareLock); + toastrel = table_open(toastrelid, AccessShareLock); /* Fetch all chunks */ - table_relation_fetch_toast_slice(toastrel, toast_pointer.va_valueid, + table_relation_fetch_toast_slice(toastrel, valueid, attrsize, sliceoffset, slicelength, result); @@ -550,10 +609,20 @@ toast_raw_datum_size(Datum value) if (VARATT_IS_EXTERNAL_ONDISK(attr)) { /* va_rawsize is the size of the original datum -- including header */ - varatt_external_oid toast_pointer; + if (VARTAG_EXTERNAL(attr) == VARTAG_ONDISK_OID8) + { + varatt_external_oid8 toast_pointer; - VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); - result = toast_pointer.va_rawsize; + VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + result = toast_pointer.va_rawsize; + } + else + { + varatt_external_oid toast_pointer; + + VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + result = toast_pointer.va_rawsize; + } } else if (VARATT_IS_EXTERNAL_INDIRECT(attr)) { @@ -610,10 +679,20 @@ toast_datum_size(Datum value) * compressed or not. We do not count the size of the toast pointer * ... should we? */ - varatt_external_oid toast_pointer; + if (VARTAG_EXTERNAL(attr) == VARTAG_ONDISK_OID8) + { + varatt_external_oid8 toast_pointer; - VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); - result = VARATT_EXTERNAL_OID_GET_EXTSIZE(toast_pointer); + VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + result = VARATT_EXTERNAL_OID8_GET_EXTSIZE(toast_pointer); + } + else + { + varatt_external_oid toast_pointer; + + VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + result = VARATT_EXTERNAL_OID_GET_EXTSIZE(toast_pointer); + } } else if (VARATT_IS_EXTERNAL_INDIRECT(attr)) { diff --git a/src/backend/access/common/toast_compression.c b/src/backend/access/common/toast_compression.c index 3b1bd6519261..200cee6be705 100644 --- a/src/backend/access/common/toast_compression.c +++ b/src/backend/access/common/toast_compression.c @@ -260,7 +260,16 @@ toast_get_compression_id(varlena *attr) * the external toast pointer. If compressed inline, fetch it from the * toast compression header. */ - if (VARATT_IS_EXTERNAL_ONDISK(attr)) + if (VARATT_IS_EXTERNAL_ONDISK_OID8(attr)) + { + varatt_external_oid8 toast_pointer; + + VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + + if (VARATT_EXTERNAL_OID8_IS_COMPRESSED(toast_pointer)) + cmid = VARATT_EXTERNAL_OID8_GET_COMPRESS_METHOD(toast_pointer); + } + else if (VARATT_IS_EXTERNAL_ONDISK_OID(attr)) { varatt_external_oid toast_pointer; diff --git a/src/backend/access/common/toast_internals.c b/src/backend/access/common/toast_internals.c index 2c180491c2ff..84c4361af23d 100644 --- a/src/backend/access/common/toast_internals.c +++ b/src/backend/access/common/toast_internals.c @@ -125,7 +125,6 @@ toast_save_datum(Relation rel, Datum value, TupleDesc toasttupDesc; CommandId mycid = GetCurrentCommandId(true); varlena *result; - varatt_external_oid toast_pointer; int32 chunk_seq = 0; char *data_p; int32 data_todo; @@ -133,9 +132,17 @@ toast_save_datum(Relation rel, Datum value, int num_indexes; int validIndex; Oid toast_typid = get_atttype(rel->rd_rel->reltoastrelid, 1); + int32 max_chunk_size; + + /* Fields that will be assembled into the TOAST pointer at the end */ + int32 va_rawsize; + uint32 va_extinfo; + Oid8 va_valueid; + Oid va_toastrelid; Assert(!VARATT_IS_EXTERNAL(dval)); Assert(OidIsValid(toast_typid)); + Assert(toast_typid == OIDOID || toast_typid == OID8OID); /* * Open the toast relation and its indexes. We can use the index to check @@ -165,28 +172,32 @@ toast_save_datum(Relation rel, Datum value, { data_p = VARDATA_SHORT(dval); data_todo = VARSIZE_SHORT(dval) - VARHDRSZ_SHORT; - toast_pointer.va_rawsize = data_todo + VARHDRSZ; /* as if not short */ - toast_pointer.va_extinfo = data_todo; + va_rawsize = data_todo + VARHDRSZ; /* as if not short */ + va_extinfo = data_todo; } else if (VARATT_IS_COMPRESSED(dval)) { + uint32 cmid = VARDATA_COMPRESSED_GET_COMPRESS_METHOD(dval); + data_p = VARDATA(dval); data_todo = VARSIZE(dval) - VARHDRSZ; /* rawsize in a compressed datum is just the size of the payload */ - toast_pointer.va_rawsize = VARDATA_COMPRESSED_GET_EXTSIZE(dval) + VARHDRSZ; + va_rawsize = VARDATA_COMPRESSED_GET_EXTSIZE(dval) + VARHDRSZ; /* set external size and compression method */ - VARATT_EXTERNAL_SET_SIZE_AND_COMPRESS_METHOD(toast_pointer, data_todo, - VARDATA_COMPRESSED_GET_COMPRESS_METHOD(dval)); + Assert(cmid == TOAST_PGLZ_COMPRESSION_ID || + cmid == TOAST_LZ4_COMPRESSION_ID); + va_extinfo = data_todo | (cmid << VARLENA_EXTSIZE_BITS); + /* Assert that the numbers look like it's compressed */ - Assert(VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer)); + Assert((va_extinfo & VARLENA_EXTSIZE_MASK) < (Size) (va_rawsize - VARHDRSZ)); } else { data_p = VARDATA(dval); data_todo = VARSIZE(dval) - VARHDRSZ; - toast_pointer.va_rawsize = VARSIZE(dval); - toast_pointer.va_extinfo = data_todo; + va_rawsize = VARSIZE(dval); + va_extinfo = data_todo; } /* @@ -198,13 +209,14 @@ toast_save_datum(Relation rel, Datum value, * if we have to substitute such an OID. */ if (OidIsValid(rel->rd_toastoid)) - toast_pointer.va_toastrelid = rel->rd_toastoid; + va_toastrelid = rel->rd_toastoid; else - toast_pointer.va_toastrelid = RelationGetRelid(toastrel); + va_toastrelid = RelationGetRelid(toastrel); /* - * Choose a new value to use as the value ID for this toast value, be it - * for OID or OID8 TOAST relations. + * Choose a new value to use as the value ID for this toast value, and + * determine the maximum chunk size based on the TOAST table's chunk_id + * type. * * Normally we just choose an unused value within the toast table. But * during table-rewriting operations where we are preserving an existing @@ -213,84 +225,150 @@ toast_save_datum(Relation rel, Datum value, * toast table, re-use its value ID. If we didn't have a prior external * value (which is a corner case, but possible if the table's attstorage * options have been changed), we have to pick a value ID that doesn't - * conflict with either new or existing toast value IDs. If the TOAST - * table uses 8-byte value IDs, we should not really care much about - * that. + * conflict with either new or existing toast value IDs. For Oid8 tables, + * value conflicts are not a concern. */ - if (!OidIsValid(rel->rd_toastoid)) + if (toast_typid == OID8OID) { - /* normal case: just choose an unused OID */ - if (toast_typid == OIDOID) - toast_pointer.va_valueid = - GetNewOidWithIndex(toastrel, - RelationGetRelid(toastidxs[validIndex]), - (AttrNumber) 1); - else if (toast_typid == OID8OID) - toast_pointer.va_valueid = GetNewObjectId8(); + if (!OidIsValid(rel->rd_toastoid)) + { + /* normal case: just choose a new Oid8 */ + va_valueid = GetNewObjectId8(); + } else - Assert(false); + { + /* rewrite case: check to see if value was in old toast table */ + va_valueid = InvalidOid8; + if (oldexternal != NULL) + { + Assert(VARATT_IS_EXTERNAL_ONDISK(oldexternal)); + + /* + * Only consider the old pointer if its vartag matches the + * kind of pointer we are about to build. + */ + if (VARATT_IS_EXTERNAL_ONDISK_OID8(oldexternal)) + { + varatt_external_oid8 old_toast_pointer; + + /* Must copy to access aligned fields */ + VARATT_EXTERNAL_GET_POINTER(old_toast_pointer, oldexternal); + if (old_toast_pointer.va_toastrelid == rel->rd_toastoid) + { + /* + * This value came from the old toast table; reuse its + * ID. + */ + va_valueid = VARATT_EXTERNAL_OID8_GET_VALUEID(old_toast_pointer); + + /* + * There is a corner case here: the table rewrite + * might have to copy both live and recently-dead + * versions of a row, and those versions could easily + * reference the same toast value. When we copy the + * second or later version of such a row, reusing the + * OID will mean we select an OID that's already in the + * new toast table. Check for that, and if so, just + * fall through without writing the data again. + * + * While annoying and ugly-looking, this is a good + * thing because it ensures that we wind up with only + * one copy of the toast value when there is only one + * copy in the old toast table. Before we detected + * this case, we'd have made multiple copies, wasting + * space; and what's worse, the copies belonging to + * already-deleted heap tuples would not be reclaimed + * by VACUUM. + */ + if (toastrel_valueid_exists(toastrel, va_valueid)) + { + /* + * Match, so short-circuit the data storage loop + * below. + */ + data_todo = 0; + } + } + } + } + if (va_valueid == InvalidOid8) + { + /* + * New value. For Oid8 we just pick a new ID without worrying + * about conflicts. + */ + va_valueid = GetNewObjectId8(); + } + } + + max_chunk_size = TOAST_OID8_MAX_CHUNK_SIZE; } else { - /* rewrite case: check to see if value was in old toast table */ - toast_pointer.va_valueid = InvalidOid; - if (oldexternal != NULL) + if (!OidIsValid(rel->rd_toastoid)) { - varatt_external_oid old_toast_pointer; - - Assert(VARATT_IS_EXTERNAL_ONDISK(oldexternal)); - /* Must copy to access aligned fields */ - VARATT_EXTERNAL_GET_POINTER(old_toast_pointer, oldexternal); - if (old_toast_pointer.va_toastrelid == rel->rd_toastoid) + /* normal case: just choose an unused OID */ + va_valueid = + GetNewOidWithIndex(toastrel, + RelationGetRelid(toastidxs[validIndex]), + (AttrNumber) 1); + } + else + { + /* rewrite case: check to see if value was in old toast table */ + va_valueid = InvalidOid; + if (oldexternal != NULL) { - /* This value came from the old toast table; reuse its OID */ - toast_pointer.va_valueid = old_toast_pointer.va_valueid; + Assert(VARATT_IS_EXTERNAL_ONDISK(oldexternal)); - /* - * There is a corner case here: the table rewrite might have - * to copy both live and recently-dead versions of a row, and - * those versions could easily reference the same toast value. - * When we copy the second or later version of such a row, - * reusing the OID will mean we select an OID that's already - * in the new toast table. Check for that, and if so, just - * fall through without writing the data again. - * - * While annoying and ugly-looking, this is a good thing - * because it ensures that we wind up with only one copy of - * the toast value when there is only one copy in the old - * toast table. Before we detected this case, we'd have made - * multiple copies, wasting space; and what's worse, the - * copies belonging to already-deleted heap tuples would not - * be reclaimed by VACUUM. - */ - if (toastrel_valueid_exists(toastrel, - toast_pointer.va_valueid)) + /* Only reuse a pointer of the matching type; same as above */ + if (VARATT_IS_EXTERNAL_ONDISK_OID(oldexternal)) { - /* Match, so short-circuit the data storage loop below */ - data_todo = 0; + varatt_external_oid old_toast_pointer; + + /* Must copy to access aligned fields */ + VARATT_EXTERNAL_GET_POINTER(old_toast_pointer, oldexternal); + if (old_toast_pointer.va_toastrelid == rel->rd_toastoid) + { + /* + * This value came from the old toast table; reuse its + * OID. + */ + va_valueid = old_toast_pointer.va_valueid; + + /* + * Corner case during table rewrite with multiple + * versions of the same row. See above for details. + */ + if (toastrel_valueid_exists(toastrel, va_valueid)) + { + /* + * Match, so short-circuit the data storage loop + * below. + */ + data_todo = 0; + } + } } } - } - if (toast_pointer.va_valueid == InvalidOid) - { - /* - * new value; must choose a value that doesn't conflict in either - * old or new toast table. - */ - if (toast_typid == OIDOID) + if (va_valueid == InvalidOid) { + /* + * new value; must choose a value that doesn't conflict in + * either old or new toast table. + */ do { - toast_pointer.va_valueid = + va_valueid = GetNewOidWithIndex(toastrel, RelationGetRelid(toastidxs[validIndex]), (AttrNumber) 1); } while (toastid_valueid_exists(rel->rd_toastoid, - toast_pointer.va_valueid)); + va_valueid)); } - else if (toast_typid == OID8OID) - toast_pointer.va_valueid = GetNewObjectId8(); } + + max_chunk_size = TOAST_OID_MAX_CHUNK_SIZE; } /* @@ -314,15 +392,15 @@ toast_save_datum(Relation rel, Datum value, /* * Calculate the size of this chunk */ - chunk_size = Min(TOAST_OID_MAX_CHUNK_SIZE, data_todo); + chunk_size = Min(max_chunk_size, data_todo); /* * Build a tuple and store it */ - if (toast_typid == OIDOID) - t_values[0] = ObjectIdGetDatum(toast_pointer.va_valueid); - else if (toast_typid == OID8OID) - t_values[0] = ObjectId8GetDatum(toast_pointer.va_valueid); + if (toast_typid == OID8OID) + t_values[0] = ObjectId8GetDatum(va_valueid); + else + t_values[0] = ObjectIdGetDatum((Oid) va_valueid); t_values[1] = Int32GetDatum(chunk_seq++); SET_VARSIZE(&chunk_data, chunk_size + VARHDRSZ); memcpy(VARDATA(&chunk_data), data_p, chunk_size); @@ -376,11 +454,34 @@ toast_save_datum(Relation rel, Datum value, table_close(toastrel, NoLock); /* - * Create the TOAST pointer value that we'll return + * Create the TOAST pointer value that we'll return. */ - result = (varlena *) palloc(TOAST_OID_POINTER_SIZE); - SET_VARTAG_EXTERNAL(result, VARTAG_ONDISK_OID); - memcpy(VARDATA_EXTERNAL(result), &toast_pointer, sizeof(toast_pointer)); + if (toast_typid == OID8OID) + { + varatt_external_oid8 toast_pointer; + + toast_pointer.va_rawsize = va_rawsize; + toast_pointer.va_extinfo = va_extinfo; + VARATT_EXTERNAL_OID8_SET_VALUEID(&toast_pointer, va_valueid); + toast_pointer.va_toastrelid = va_toastrelid; + + result = (varlena *) palloc(TOAST_OID8_POINTER_SIZE); + SET_VARTAG_EXTERNAL(result, VARTAG_ONDISK_OID8); + memcpy(VARDATA_EXTERNAL(result), &toast_pointer, sizeof(toast_pointer)); + } + else + { + varatt_external_oid toast_pointer; + + toast_pointer.va_rawsize = va_rawsize; + toast_pointer.va_extinfo = va_extinfo; + toast_pointer.va_valueid = (Oid) va_valueid; + toast_pointer.va_toastrelid = va_toastrelid; + + result = (varlena *) palloc(TOAST_OID_POINTER_SIZE); + SET_VARTAG_EXTERNAL(result, VARTAG_ONDISK_OID); + memcpy(VARDATA_EXTERNAL(result), &toast_pointer, sizeof(toast_pointer)); + } return PointerGetDatum(result); } @@ -395,7 +496,6 @@ void toast_delete_datum(Relation rel, Datum value, bool is_speculative) { varlena *attr = (varlena *) DatumGetPointer(value); - varatt_external_oid toast_pointer; Relation toastrel; Relation *toastidxs; ScanKeyData toastkey; @@ -403,42 +503,75 @@ toast_delete_datum(Relation rel, Datum value, bool is_speculative) HeapTuple toasttup; int num_indexes; int validIndex; - Oid toast_typid; + Oid toastrelid; + vartag_external tag; if (!VARATT_IS_EXTERNAL_ONDISK(attr)) return; - /* Must copy to access aligned fields */ - VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); - /* - * Open the toast relation and its indexes + * Determine the pointer type from the datum's vartag and extract the + * toast relation OID and value ID accordingly. The vartag tells us + * everything we need - no TOAST table schema lookup required. */ - toastrel = table_open(toast_pointer.va_toastrelid, RowExclusiveLock); - toast_typid = TupleDescAttr(toastrel->rd_att, 0)->atttypid; - Assert(toast_typid == OIDOID || toast_typid == OID8OID); + tag = VARTAG_EXTERNAL(attr); - /* Fetch valid relation used for process */ - validIndex = toast_open_indexes(toastrel, - RowExclusiveLock, - &toastidxs, - &num_indexes); + if (tag == VARTAG_ONDISK_OID8) + { + varatt_external_oid8 toast_pointer8; - /* - * Setup a scan key to find chunks with matching va_valueid - */ - if (toast_typid == OIDOID) + /* Must copy to access aligned fields */ + VARATT_EXTERNAL_GET_POINTER(toast_pointer8, attr); + toastrelid = toast_pointer8.va_toastrelid; + + /* + * Open the toast relation and its indexes + */ + toastrel = table_open(toastrelid, RowExclusiveLock); + + /* Fetch valid relation used for process */ + validIndex = toast_open_indexes(toastrel, + RowExclusiveLock, + &toastidxs, + &num_indexes); + + /* + * Setup a scan key to find chunks with matching va_valueid + */ + ScanKeyInit(&toastkey, + (AttrNumber) 1, + BTEqualStrategyNumber, F_OID8EQ, + ObjectId8GetDatum(VARATT_EXTERNAL_OID8_GET_VALUEID(toast_pointer8))); + } + else + { + varatt_external_oid toast_pointer; + + Assert(tag == VARTAG_ONDISK_OID); + + /* Must copy to access aligned fields */ + VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + toastrelid = toast_pointer.va_toastrelid; + + /* + * Open the toast relation and its indexes + */ + toastrel = table_open(toastrelid, RowExclusiveLock); + + /* Fetch valid relation used for process */ + validIndex = toast_open_indexes(toastrel, + RowExclusiveLock, + &toastidxs, + &num_indexes); + + /* + * Setup a scan key to find chunks with matching va_valueid + */ ScanKeyInit(&toastkey, (AttrNumber) 1, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(toast_pointer.va_valueid)); - else if (toast_typid == OID8OID) - ScanKeyInit(&toastkey, - (AttrNumber) 1, - BTEqualStrategyNumber, F_OID8EQ, - ObjectId8GetDatum(toast_pointer.va_valueid)); - else - Assert(false); + } /* * Find all the chunks. (We don't actually care whether we see them in @@ -499,18 +632,16 @@ toastrel_valueid_exists(Relation toastrel, Oid8 valueid) /* * Setup a scan key to find chunks with matching va_valueid */ - if (toast_typid == OIDOID) - ScanKeyInit(&toastkey, - (AttrNumber) 1, - BTEqualStrategyNumber, F_OIDEQ, - ObjectIdGetDatum(valueid)); - else if (toast_typid == OID8OID) + if (toast_typid == OID8OID) ScanKeyInit(&toastkey, (AttrNumber) 1, BTEqualStrategyNumber, F_OID8EQ, ObjectId8GetDatum(valueid)); else - Assert(false); + ScanKeyInit(&toastkey, + (AttrNumber) 1, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum((Oid) valueid)); /* * Is there any such chunk? diff --git a/src/backend/access/heap/heaptoast.c b/src/backend/access/heap/heaptoast.c index a93e98788da7..31e6115de201 100644 --- a/src/backend/access/heap/heaptoast.c +++ b/src/backend/access/heap/heaptoast.c @@ -650,7 +650,10 @@ heap_fetch_toast_slice(Relation toastrel, Oid8 valueid, int32 attrsize, toast_typid = TupleDescAttr(toastrel->rd_att, 0)->atttypid; Assert(toast_typid == OIDOID || toast_typid == OID8OID); - max_chunk_size = TOAST_OID_MAX_CHUNK_SIZE; + if (toast_typid == OID8OID) + max_chunk_size = TOAST_OID8_MAX_CHUNK_SIZE; + else + max_chunk_size = TOAST_OID_MAX_CHUNK_SIZE; totalchunks = ((attrsize - 1) / max_chunk_size) + 1; startchunk = sliceoffset / max_chunk_size; @@ -658,18 +661,16 @@ heap_fetch_toast_slice(Relation toastrel, Oid8 valueid, int32 attrsize, Assert(endchunk <= totalchunks); /* Set up a scan key to fetch from the index. */ - if (toast_typid == OIDOID) - ScanKeyInit(&toastkey[0], - (AttrNumber) 1, - BTEqualStrategyNumber, F_OIDEQ, - ObjectIdGetDatum(valueid)); - else if (toast_typid == OID8OID) + if (toast_typid == OID8OID) ScanKeyInit(&toastkey[0], (AttrNumber) 1, BTEqualStrategyNumber, F_OID8EQ, ObjectId8GetDatum(valueid)); else - Assert(false); + ScanKeyInit(&toastkey[0], + (AttrNumber) 1, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum((Oid) valueid)); /* * No additional condition if fetching all chunks. Otherwise, use an diff --git a/src/backend/access/table/toast_helper.c b/src/backend/access/table/toast_helper.c index 6f2d691cc681..6c7257f5f94b 100644 --- a/src/backend/access/table/toast_helper.c +++ b/src/backend/access/table/toast_helper.c @@ -69,12 +69,16 @@ toast_tuple_init(ToastTupleContext *ttc) /* * If the old value is stored on disk, check if it has changed so * we have to delete it later. + * + * Note that TOAST pointers could have different vartags, for oid + * or oid8, and these can have a different sizes. */ if (att->attlen == -1 && !ttc->ttc_oldisnull[i] && VARATT_IS_EXTERNAL_ONDISK(old_value)) { if (ttc->ttc_isnull[i] || !VARATT_IS_EXTERNAL_ONDISK(new_value) || + VARTAG_EXTERNAL(old_value) != VARTAG_EXTERNAL(new_value) || memcmp(old_value, new_value, VARSIZE_EXTERNAL(old_value)) != 0) { diff --git a/src/backend/catalog/toasting.c b/src/backend/catalog/toasting.c index 66d483bb1be3..57ebaec69c78 100644 --- a/src/backend/catalog/toasting.c +++ b/src/backend/catalog/toasting.c @@ -167,10 +167,21 @@ create_toast_table(Relation rel, Oid toastOid, Oid toastIndexOid, return false; value_type = RelationGetToastValueType(rel, STDRD_OPTION_TOAST_VALUE_TYPE_OID); - if (value_type == STDRD_OPTION_TOAST_VALUE_TYPE_OID) - toast_chunkid_typid = OIDOID; - else if (value_type == STDRD_OPTION_TOAST_VALUE_TYPE_OID8) - toast_chunkid_typid = OID8OID; + + /* no default clause to catch new values added */ + switch (value_type) + { + case STDRD_OPTION_TOAST_VALUE_TYPE_OID: + toast_chunkid_typid = OIDOID; + break; + case STDRD_OPTION_TOAST_VALUE_TYPE_OID8: + toast_chunkid_typid = OID8OID; + break; + case STDRD_OPTION_TOAST_VALUE_TYPE_INVALID: + elog(ERROR, "unexpected toast_value_type value %d", + value_type); + break; + } } else { diff --git a/src/backend/replication/logical/reorderbuffer.c b/src/backend/replication/logical/reorderbuffer.c index dd4b6e8f7f45..086ef31e4f2f 100644 --- a/src/backend/replication/logical/reorderbuffer.c +++ b/src/backend/replication/logical/reorderbuffer.c @@ -5176,12 +5176,14 @@ ReorderBufferToastReplace(ReorderBuffer *rb, ReorderBufferTXN *txn, /* va_rawsize is the size of the original datum -- including header */ varatt_external_oid toast_pointer; + varatt_external_oid8 toast_pointer8; varatt_indirect redirect_pointer; varlena *new_datum = NULL; varlena *reconstructed; dlist_iter it; Size data_done = 0; Oid8 toast_valueid; + int32 rawsize; if (attr->attisdropped) continue; @@ -5198,11 +5200,21 @@ ReorderBufferToastReplace(ReorderBuffer *rb, ReorderBufferTXN *txn, varlena_pointer = (varlena *) DatumGetPointer(attrs[natt]); /* no need to do anything if the tuple isn't external */ - if (!VARATT_IS_EXTERNAL(varlena_pointer)) + if (!VARATT_IS_EXTERNAL_ONDISK(varlena_pointer)) continue; - VARATT_EXTERNAL_GET_POINTER(toast_pointer, varlena_pointer); - toast_valueid = toast_pointer.va_valueid; + if (VARATT_IS_EXTERNAL_ONDISK_OID8(varlena_pointer)) + { + VARATT_EXTERNAL_GET_POINTER(toast_pointer8, varlena_pointer); + toast_valueid = VARATT_EXTERNAL_OID8_GET_VALUEID(toast_pointer8); + rawsize = toast_pointer8.va_rawsize; + } + else + { + VARATT_EXTERNAL_GET_POINTER(toast_pointer, varlena_pointer); + toast_valueid = toast_pointer.va_valueid; + rawsize = toast_pointer.va_rawsize; + } /* * Check whether the toast tuple changed, replace if so. @@ -5220,7 +5232,7 @@ ReorderBufferToastReplace(ReorderBuffer *rb, ReorderBufferTXN *txn, free[natt] = true; - reconstructed = palloc0(toast_pointer.va_rawsize); + reconstructed = palloc0(rawsize); ent->reconstructed = reconstructed; @@ -5245,13 +5257,28 @@ ReorderBufferToastReplace(ReorderBuffer *rb, ReorderBufferTXN *txn, VARSIZE(chunk) - VARHDRSZ); data_done += VARSIZE(chunk) - VARHDRSZ; } - Assert(data_done == VARATT_EXTERNAL_OID_GET_EXTSIZE(toast_pointer)); - /* make sure its marked as compressed or not */ - if (VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer)) - SET_VARSIZE_COMPRESSED(reconstructed, data_done + VARHDRSZ); + /* Verify size and set compression status based on pointer type */ + if (VARATT_IS_EXTERNAL_ONDISK_OID8(varlena_pointer)) + { + Assert(data_done == VARATT_EXTERNAL_OID8_GET_EXTSIZE(toast_pointer8)); + + /* make sure its marked as compressed or not */ + if (VARATT_EXTERNAL_OID8_IS_COMPRESSED(toast_pointer8)) + SET_VARSIZE_COMPRESSED(reconstructed, data_done + VARHDRSZ); + else + SET_VARSIZE(reconstructed, data_done + VARHDRSZ); + } else - SET_VARSIZE(reconstructed, data_done + VARHDRSZ); + { + Assert(data_done == VARATT_EXTERNAL_OID_GET_EXTSIZE(toast_pointer)); + + /* make sure its marked as compressed or not */ + if (VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer)) + SET_VARSIZE_COMPRESSED(reconstructed, data_done + VARHDRSZ); + else + SET_VARSIZE(reconstructed, data_done + VARHDRSZ); + } memset(&redirect_pointer, 0, sizeof(redirect_pointer)); redirect_pointer.pointer = reconstructed; diff --git a/src/backend/utils/adt/varlena.c b/src/backend/utils/adt/varlena.c index 1e264904a6e1..e104a6308d65 100644 --- a/src/backend/utils/adt/varlena.c +++ b/src/backend/utils/adt/varlena.c @@ -4261,7 +4261,7 @@ pg_column_toast_chunk_id(PG_FUNCTION_ARGS) { int typlen; varlena *attr; - varatt_external_oid toast_pointer; + Oid8 result; /* On first call, get the input type's typlen, and save at *fn_extra */ if (fcinfo->flinfo->fn_extra == NULL) @@ -4288,9 +4288,22 @@ pg_column_toast_chunk_id(PG_FUNCTION_ARGS) if (!VARATT_IS_EXTERNAL_ONDISK(attr)) PG_RETURN_NULL(); - VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + if (VARATT_IS_EXTERNAL_ONDISK_OID8(attr)) + { + varatt_external_oid8 toast_pointer; - PG_RETURN_OID8(toast_pointer.va_valueid); + VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + result = VARATT_EXTERNAL_OID8_GET_VALUEID(toast_pointer); + } + else + { + varatt_external_oid toast_pointer; + + VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + result = toast_pointer.va_valueid; + } + + PG_RETURN_OID8(result); } /* diff --git a/doc/src/sgml/ref/create_table.sgml b/doc/src/sgml/ref/create_table.sgml index 3ed8f3581177..158f79e7135f 100644 --- a/doc/src/sgml/ref/create_table.sgml +++ b/doc/src/sgml/ref/create_table.sgml @@ -1661,14 +1661,23 @@ WITH ( MODULUS numeric_literal, REM - The toast_value_type specifies the attribute type of - chunk_id used when initially creating a toast - relation for this table. - By default this parameter is oid, to assign - oid as attribute type to chunk_id. - This parameter can be set to oid8 to use oid8 - as attribute type for chunk_id. - This parameter cannot be set for TOAST tables. + Specifies the attribute type of chunk_id to use when + creating a TOAST relation for this table. The + default is oid; oid8 assigns + oid8 instead, which allows a larger number of distinct + TOASTed values at the price of four extra bytes per + out-of-line pointer. This parameter cannot be set for + TOAST tables. + + + The value is only consulted when the TOAST relation + is created. Changing it afterwards has no effect on an existing + TOAST relation, and a table rewrite such as + CLUSTER or + VACUUM FULL + preserves the type already in use. A dump and restore, on the other + hand, recreates the TOAST relation and therefore + applies the current value of this parameter. diff --git a/doc/src/sgml/storage.sgml b/doc/src/sgml/storage.sgml index 21bd67c11ea0..3561a6a67ece 100644 --- a/doc/src/sgml/storage.sgml +++ b/doc/src/sgml/storage.sgml @@ -417,7 +417,9 @@ described in more detail below. Out-of-line values are divided (after compression if used) into chunks of at -most TOAST_OID_MAX_CHUNK_SIZE bytes (by default this value is chosen +most TOAST_OID_MAX_CHUNK_SIZE or +TOAST_OID8_MAX_CHUNK_SIZE bytes depending on the +chunk_id type (by default this value is chosen so that four chunk rows will fit on a page, making it about 2000 bytes). Each chunk is stored as a separate row in the TOAST table belonging to the owning table. Every @@ -434,8 +436,10 @@ retrieval of the values. A pointer datum representing an out-of-line on-disk logical datum size (original uncompressed data length), physical stored size (different if compression was applied), and the compression method used, if any. Allowing for the varlena header bytes, -the total size of an on-disk TOAST pointer datum is therefore 18 -bytes regardless of the actual size of the represented value. +the total size of an on-disk TOAST pointer datum is 18 +bytes when using an OID as chunk_id, or 22 bytes +when using an OID8 as chunk_id, regardless of the +actual size of the represented value. diff --git a/contrib/amcheck/verify_heapam.c b/contrib/amcheck/verify_heapam.c index 42f409548e1d..86dc9cf51a93 100644 --- a/contrib/amcheck/verify_heapam.c +++ b/contrib/amcheck/verify_heapam.c @@ -75,7 +75,9 @@ typedef enum SkipPages */ typedef struct ToastedAttribute { - varatt_external_oid toast_pointer; + vartag_external tag; /* VARTAG_ONDISK_OID or VARTAG_ONDISK_OID8 */ + Oid8 va_valueid; /* value ID (works for both Oid and Oid8) */ + uint32 va_extinfo; /* external size and compression method */ BlockNumber blkno; /* block in main table */ OffsetNumber offnum; /* offset in main table */ AttrNumber attnum; /* attribute in main table */ @@ -1566,9 +1568,12 @@ check_toast_tuple(HeapTuple toasttup, HeapCheckContext *ctx, int32 max_chunk_size; Oid8 toast_valueid; - max_chunk_size = TOAST_OID_MAX_CHUNK_SIZE; + toast_valueid = ta->va_valueid; + + max_chunk_size = ta->tag == VARTAG_ONDISK_OID8 + ? TOAST_OID8_MAX_CHUNK_SIZE + : TOAST_OID_MAX_CHUNK_SIZE; last_chunk_seq = (extsize - 1) / max_chunk_size; - toast_valueid = ta->toast_pointer.va_valueid; /* Sanity-check the sequence number. */ chunk_seq = DatumGetInt32(fastgetattr(toasttup, 2, @@ -1672,8 +1677,10 @@ check_tuple_attribute(HeapCheckContext *ctx) char *tp; /* pointer to the tuple data */ uint16 infomask; Oid8 toast_pointer_valueid; + int32 va_rawsize; + uint32 va_extinfo; CompactAttribute *thisatt; - varatt_external_oid toast_pointer; + vartag_external va_tag_value; infomask = ctx->tuphdr->t_infomask; thisatt = TupleDescCompactAttr(RelationGetDescr(ctx->rel), ctx->attnum); @@ -1732,7 +1739,7 @@ check_tuple_attribute(HeapCheckContext *ctx) { uint8 va_tag = VARTAG_EXTERNAL(tp + ctx->offset); - if (va_tag != VARTAG_ONDISK_OID) + if (va_tag != VARTAG_ONDISK_OID && va_tag != VARTAG_ONDISK_OID8) { report_corruption(ctx, psprintf("toasted attribute has unexpected TOAST tag %u", @@ -1777,26 +1784,46 @@ check_tuple_attribute(HeapCheckContext *ctx) /* It is external, and we're looking at a page on disk */ /* - * Must copy attr into toast_pointer for alignment considerations + * Must copy attr into toast_pointer for alignment considerations. + * Branch on the tag to determine which pointer type to extract. */ - VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); - toast_pointer_valueid = toast_pointer.va_valueid; + va_tag_value = VARTAG_EXTERNAL(attr); + if (va_tag_value == VARTAG_ONDISK_OID8) + { + varatt_external_oid8 toast_pointer8; + + /* Must copy to access aligned fields */ + VARATT_EXTERNAL_GET_POINTER(toast_pointer8, attr); + toast_pointer_valueid = VARATT_EXTERNAL_OID8_GET_VALUEID(toast_pointer8); + va_rawsize = toast_pointer8.va_rawsize; + va_extinfo = toast_pointer8.va_extinfo; + } + else + { + varatt_external_oid toast_pointer; + + /* Must copy to access aligned fields */ + VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + toast_pointer_valueid = toast_pointer.va_valueid; + va_rawsize = toast_pointer.va_rawsize; + va_extinfo = toast_pointer.va_extinfo; + } /* Toasted attributes too large to be untoasted should never be stored */ - if (toast_pointer.va_rawsize > VARLENA_SIZE_LIMIT) + if (va_rawsize > VARLENA_SIZE_LIMIT) report_corruption(ctx, psprintf("toast value " OID8_FORMAT " rawsize %d exceeds limit %d", toast_pointer_valueid, - toast_pointer.va_rawsize, + va_rawsize, VARLENA_SIZE_LIMIT)); - if (VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer)) + if ((va_extinfo & VARLENA_EXTSIZE_MASK) < (Size) (va_rawsize - VARHDRSZ)) { ToastCompressionId cmid; bool valid = false; /* Compressed attributes should have a valid compression method */ - cmid = TOAST_COMPRESS_METHOD(&toast_pointer); + cmid = va_extinfo >> VARLENA_EXTSIZE_BITS; switch (cmid) { /* List of all valid compression method IDs */ @@ -1850,7 +1877,10 @@ check_tuple_attribute(HeapCheckContext *ctx) ta = palloc0_object(ToastedAttribute); - VARATT_EXTERNAL_GET_POINTER(ta->toast_pointer, attr); + /* The pointer has already been decoded above, just reuse it */ + ta->tag = va_tag_value; + ta->va_valueid = toast_pointer_valueid; + ta->va_extinfo = va_extinfo; ta->blkno = ctx->blkno; ta->offnum = ctx->offnum; ta->attnum = ctx->attnum; @@ -1878,29 +1908,33 @@ check_toasted_attribute(HeapCheckContext *ctx, ToastedAttribute *ta) int32 last_chunk_seq; int32 max_chunk_size; Oid8 toast_valueid; - Oid toast_typid; - toast_typid = TupleDescAttr(ctx->toast_rel->rd_att, 0)->atttypid; - max_chunk_size = TOAST_OID_MAX_CHUNK_SIZE; - - extsize = VARATT_EXTERNAL_OID_GET_EXTSIZE(ta->toast_pointer); - last_chunk_seq = (extsize - 1) / max_chunk_size; + toast_valueid = ta->va_valueid; + extsize = ta->va_extinfo & VARLENA_EXTSIZE_MASK; /* * Setup a scan key to find chunks in toast table with matching va_valueid */ - if (toast_typid == OIDOID) - ScanKeyInit(&toastkey, - (AttrNumber) 1, - BTEqualStrategyNumber, F_OIDEQ, - ObjectIdGetDatum(ta->toast_pointer.va_valueid)); - else if (toast_typid == OID8OID) + if (ta->tag == VARTAG_ONDISK_OID8) + { + max_chunk_size = TOAST_OID8_MAX_CHUNK_SIZE; + ScanKeyInit(&toastkey, (AttrNumber) 1, BTEqualStrategyNumber, F_OID8EQ, - ObjectId8GetDatum(ta->toast_pointer.va_valueid)); + ObjectId8GetDatum(toast_valueid)); + } else - Assert(false); + { + max_chunk_size = TOAST_OID_MAX_CHUNK_SIZE; + + ScanKeyInit(&toastkey, + (AttrNumber) 1, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum((Oid) toast_valueid)); + } + + last_chunk_seq = (extsize - 1) / max_chunk_size; /* * Check if any chunks for this toasted object exist in the toast table, @@ -1920,8 +1954,6 @@ check_toasted_attribute(HeapCheckContext *ctx, ToastedAttribute *ta) } systable_endscan_ordered(toastscan); - toast_valueid = ta->toast_pointer.va_valueid; - if (!found_toasttup) report_toast_corruption(ctx, ta, psprintf("toast value " OID8_FORMAT " not found in toast table", -- 2.55.0