From e0fd92d5d4dc1e3a7a26601cebaad97a45e154ac Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Mon, 7 Sep 2026 16:34:22 +0900 Subject: [PATCH v14 8/9] Add support for TOAST pointers as oid8 This introduces a new varlena external pointer structure, varatt_external_oid8, that carries a 64-bit value ID (Oid8) for TOAST tables using oid8 as their chunk_id type. XXX: Catalog version bump required. --- src/include/access/detoast.h | 3 + src/include/access/heaptoast.h | 16 +- src/include/varatt.h | 82 ++++- src/backend/access/common/detoast.c | 171 +++++++--- src/backend/access/common/reloptions.c | 2 +- src/backend/access/common/toast_compression.c | 11 +- src/backend/access/common/toast_internals.c | 314 ++++++++++++------ src/backend/access/heap/heaptoast.c | 5 +- src/backend/access/table/toast_helper.c | 4 + .../replication/logical/reorderbuffer.c | 44 ++- src/backend/utils/adt/varlena.c | 19 +- src/test/regress/expected/compression.out | 41 +++ src/test/regress/expected/strings.out | 43 +++ src/test/regress/sql/compression.sql | 22 ++ src/test/regress/sql/strings.sql | 25 ++ doc/src/sgml/storage.sgml | 10 +- contrib/amcheck/verify_heapam.c | 103 ++++-- 17 files changed, 702 insertions(+), 213 deletions(-) diff --git a/src/include/access/detoast.h b/src/include/access/detoast.h index 2b1428b1e81c..f065ea034e98 100644 --- a/src/include/access/detoast.h +++ b/src/include/access/detoast.h @@ -30,6 +30,9 @@ do { \ /* Size of an EXTERNAL datum that contains a standard TOAST pointer */ #define TOAST_OID_POINTER_SIZE (VARHDRSZ_EXTERNAL + sizeof(varatt_external_oid)) +/* Size of an EXTERNAL datum that contains an Oid8 TOAST pointer */ +#define TOAST_OID8_POINTER_SIZE (VARHDRSZ_EXTERNAL + sizeof(varatt_external_oid8)) + /* Size of an EXTERNAL datum that contains an indirection pointer */ #define INDIRECT_POINTER_SIZE (VARHDRSZ_EXTERNAL + sizeof(varatt_indirect)) diff --git a/src/include/access/heaptoast.h b/src/include/access/heaptoast.h index aaae6e0fef69..b7ef69a8f56b 100644 --- a/src/include/access/heaptoast.h +++ b/src/include/access/heaptoast.h @@ -69,13 +69,14 @@ /* * When we store an oversize datum externally, we divide it into chunks - * containing at most TOAST_OID_MAX_CHUNK_SIZE data bytes. This number *must* - * be small enough that the completed toast-table tuple (including the - * ID and sequence fields and all overhead) will fit on a page. + * containing at most TOAST_OID_MAX_CHUNK_SIZE or TOAST_OID8_MAX_CHUNK_SIZE + * data bytes, depending on the chunk_id type of the TOAST table. These + * numbers *must* be small enough that the completed toast-table tuple + * (including the ID and sequence fields and all overhead) will fit on a page. * The coding here sets the size on the theory that we want to fit * EXTERN_TUPLES_PER_PAGE tuples of maximum size onto a page. * - * NB: Changing TOAST_OID_MAX_CHUNK_SIZE requires an initdb. + * NB: Changing these values requires an initdb. */ #define EXTERN_TUPLES_PER_PAGE 4 /* tweak only this */ @@ -88,6 +89,13 @@ sizeof(int32) - \ VARHDRSZ) +#define TOAST_OID8_MAX_CHUNK_SIZE \ + (EXTERN_TUPLE_MAX_SIZE - \ + MAXALIGN(SizeofHeapTupleHeader) - \ + sizeof(Oid8) - \ + sizeof(int32) - \ + VARHDRSZ) + /* ---------- * heap_toast_insert_or_update - * diff --git a/src/include/varatt.h b/src/include/varatt.h index 33979bbaaad1..bb1a59bf6cc9 100644 --- a/src/include/varatt.h +++ b/src/include/varatt.h @@ -38,6 +38,38 @@ typedef struct varatt_external_oid Oid va_toastrelid; /* RelID of TOAST table containing it */ } varatt_external_oid; +/* + * varatt_external_oid8 is a "TOAST pointer" for TOAST tables that use + * Oid8 (64-bit) as their chunk_id type. Same layout as varatt_external_oid + * except that va_valueid is split into two uint32 halves to enforce a + * stricter structure size without padding. + * + * This struct must not contain any padding, because we sometimes compare + * these pointers using memcmp. + */ +typedef struct varatt_external_oid8 +{ + int32 va_rawsize; /* Original data size (includes header) */ + uint32 va_extinfo; /* External saved size (without header) and + * compression method */ + uint32 va_valueid_lo; /* Low 32 bits of value ID */ + uint32 va_valueid_hi; /* High 32 bits of value ID */ + Oid va_toastrelid; /* RelID of TOAST table containing it */ +} varatt_external_oid8; + +StaticAssertDecl((sizeof(int32) + 3 * sizeof(uint32) + sizeof(Oid)) == + sizeof(varatt_external_oid8), + "varatt_external_oid8 should have no padding"); + +/* Helper macros to get/set the 64-bit value ID from the lo/hi fields */ +#define VARATT_EXTERNAL_OID8_GET_VALUEID(tp) \ + ((Oid8) (tp).va_valueid_lo | ((Oid8) (tp).va_valueid_hi << 32)) +#define VARATT_EXTERNAL_OID8_SET_VALUEID(tp, id) \ + do { \ + (tp).va_valueid_lo = (uint32) (id); \ + (tp).va_valueid_hi = (uint32) ((id) >> 32); \ + } while (0) + /* * These macros define the "saved size" portion of va_extinfo. Its remaining * two high-order bits identify the compression method. @@ -87,6 +119,7 @@ typedef enum vartag_external VARTAG_INDIRECT = 1, VARTAG_EXPANDED_RO = 2, VARTAG_EXPANDED_RW = 3, + VARTAG_ONDISK_OID8 = 4, VARTAG_ONDISK_OID = 18 } vartag_external; @@ -108,6 +141,8 @@ VARTAG_SIZE(vartag_external tag) return sizeof(varatt_expanded); else if (tag == VARTAG_ONDISK_OID) return sizeof(varatt_external_oid); + else if (tag == VARTAG_ONDISK_OID8) + return sizeof(varatt_external_oid8); else { Assert(false); @@ -361,7 +396,19 @@ VARATT_IS_EXTERNAL(const void *PTR) static inline bool VARATT_IS_EXTERNAL_ONDISK(const void *PTR) { - return VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_ONDISK_OID; + vartag_external tag; + + if (!VARATT_IS_EXTERNAL(PTR)) + return false; + tag = VARTAG_EXTERNAL(PTR); + return (tag == VARTAG_ONDISK_OID || tag == VARTAG_ONDISK_OID8); +} + +/* Is varlena datum a pointer to on-disk toasted data with 8-byte value ID? */ +static inline bool +VARATT_IS_EXTERNAL_ONDISK_OID8(const void *PTR) +{ + return VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_ONDISK_OID8; } /* Is varlena datum an indirect pointer? */ @@ -519,16 +566,6 @@ VARATT_EXTERNAL_OID_GET_COMPRESS_METHOD(varatt_external_oid toast_pointer) return toast_pointer.va_extinfo >> VARLENA_EXTSIZE_BITS; } -/* Set size and compress method of an externally-stored varlena datum */ -/* This has to remain a macro; beware multiple evaluations! */ -#define VARATT_EXTERNAL_SET_SIZE_AND_COMPRESS_METHOD(toast_pointer, len, cm) \ - do { \ - Assert((cm) == TOAST_PGLZ_COMPRESSION_ID || \ - (cm) == TOAST_LZ4_COMPRESSION_ID); \ - ((toast_pointer).va_extinfo = \ - (len) | ((uint32) (cm) << VARLENA_EXTSIZE_BITS)); \ - } while (0) - /* * Testing whether an externally-stored value is compressed now requires * comparing size stored in va_extinfo (the actual length of the external data) @@ -543,4 +580,27 @@ VARATT_EXTERNAL_OID_IS_COMPRESSED(varatt_external_oid toast_pointer) (Size) (toast_pointer.va_rawsize - VARHDRSZ); } +/* + * Same for external Datums; but note argument is a struct + * varatt_external_oid8. + */ +static inline Size +VARATT_EXTERNAL_OID8_GET_EXTSIZE(varatt_external_oid8 toast_pointer) +{ + return toast_pointer.va_extinfo & VARLENA_EXTSIZE_MASK; +} + +static inline uint32 +VARATT_EXTERNAL_OID8_GET_COMPRESS_METHOD(varatt_external_oid8 toast_pointer) +{ + return toast_pointer.va_extinfo >> VARLENA_EXTSIZE_BITS; +} + +static inline bool +VARATT_EXTERNAL_OID8_IS_COMPRESSED(varatt_external_oid8 toast_pointer) +{ + return VARATT_EXTERNAL_OID8_GET_EXTSIZE(toast_pointer) < + (Size) (toast_pointer.va_rawsize - VARHDRSZ); +} + #endif diff --git a/src/backend/access/common/detoast.c b/src/backend/access/common/detoast.c index a50aea7b6b12..e1919809751c 100644 --- a/src/backend/access/common/detoast.c +++ b/src/backend/access/common/detoast.c @@ -225,40 +225,51 @@ detoast_attr_slice(varlena *attr, if (VARATT_IS_EXTERNAL_ONDISK(attr)) { - varatt_external_oid toast_pointer; + int32 extsize; + uint32 compress_method; + bool is_compressed; - VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + /* Extract extsize and compression info from the appropriate pointer */ + if (VARTAG_EXTERNAL(attr) == VARTAG_ONDISK_OID8) + { + varatt_external_oid8 toast_pointer; + + VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + extsize = VARATT_EXTERNAL_OID8_GET_EXTSIZE(toast_pointer); + compress_method = VARATT_EXTERNAL_OID8_GET_COMPRESS_METHOD(toast_pointer); + is_compressed = VARATT_EXTERNAL_OID8_IS_COMPRESSED(toast_pointer); + } + else + { + varatt_external_oid toast_pointer; + + VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + extsize = VARATT_EXTERNAL_OID_GET_EXTSIZE(toast_pointer); + compress_method = VARATT_EXTERNAL_OID_GET_COMPRESS_METHOD(toast_pointer); + is_compressed = VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer); + } /* fast path for non-compressed external datums */ - if (!VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer)) + if (!is_compressed) return toast_fetch_datum_slice(attr, sliceoffset, slicelength); /* - * For compressed values, we need to fetch enough slices to decompress - * at least the requested part (when a prefix is requested). - * Otherwise, just fetch all slices. + * For compressed values, we need to fetch enough slices to + * decompress at least the requested part (when a prefix is + * requested). Otherwise, just fetch all slices. + * + * At least for now, if it's LZ4 data, we'll have to fetch the + * whole thing, because there doesn't seem to be an API call to + * determine how much compressed data we need to be sure of being + * able to decompress the required slice. */ if (slicelimit >= 0) { - int32 max_size = VARATT_EXTERNAL_OID_GET_EXTSIZE(toast_pointer); + int32 max_size = extsize; - /* - * Determine maximum amount of compressed data needed for a prefix - * of a given length (after decompression). - * - * At least for now, if it's LZ4 data, we'll have to fetch the - * whole thing, because there doesn't seem to be an API call to - * determine how much compressed data we need to be sure of being - * able to decompress the required slice. - */ - if (VARATT_EXTERNAL_OID_GET_COMPRESS_METHOD(toast_pointer) == - TOAST_PGLZ_COMPRESSION_ID) + if (compress_method == TOAST_PGLZ_COMPRESSION_ID) max_size = pglz_maximum_compressed_size(slicelimit, max_size); - /* - * Fetch enough compressed slices (compressed marker will get set - * automatically). - */ preslice = toast_fetch_datum_slice(attr, 0, max_size); } else @@ -344,23 +355,47 @@ toast_fetch_datum(varlena *attr) { Relation toastrel; varlena *result; - varatt_external_oid toast_pointer; int32 attrsize; + Oid toastrelid; + Oid8 valueid; if (!VARATT_IS_EXTERNAL_ONDISK(attr)) elog(ERROR, "toast_fetch_datum shouldn't be called for non-ondisk datums"); - /* Must copy to access aligned fields */ - VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + if (VARTAG_EXTERNAL(attr) == VARTAG_ONDISK_OID8) + { + varatt_external_oid8 toast_pointer; - attrsize = VARATT_EXTERNAL_OID_GET_EXTSIZE(toast_pointer); + /* Must copy to access aligned fields */ + VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + attrsize = VARATT_EXTERNAL_OID8_GET_EXTSIZE(toast_pointer); + toastrelid = toast_pointer.va_toastrelid; + valueid = VARATT_EXTERNAL_OID8_GET_VALUEID(toast_pointer); - result = (varlena *) palloc(attrsize + VARHDRSZ); + result = (varlena *) palloc(attrsize + VARHDRSZ); - if (VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer)) - SET_VARSIZE_COMPRESSED(result, attrsize + VARHDRSZ); + if (VARATT_EXTERNAL_OID8_IS_COMPRESSED(toast_pointer)) + SET_VARSIZE_COMPRESSED(result, attrsize + VARHDRSZ); + else + SET_VARSIZE(result, attrsize + VARHDRSZ); + } else - SET_VARSIZE(result, attrsize + VARHDRSZ); + { + varatt_external_oid toast_pointer; + + /* Must copy to access aligned fields */ + VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + attrsize = VARATT_EXTERNAL_OID_GET_EXTSIZE(toast_pointer); + toastrelid = toast_pointer.va_toastrelid; + valueid = toast_pointer.va_valueid; + + result = (varlena *) palloc(attrsize + VARHDRSZ); + + if (VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer)) + SET_VARSIZE_COMPRESSED(result, attrsize + VARHDRSZ); + else + SET_VARSIZE(result, attrsize + VARHDRSZ); + } if (attrsize == 0) return result; /* Probably shouldn't happen, but just in @@ -369,10 +404,10 @@ toast_fetch_datum(varlena *attr) /* * Open the toast relation and its indexes */ - toastrel = table_open(toast_pointer.va_toastrelid, AccessShareLock); + toastrel = table_open(toastrelid, AccessShareLock); /* Fetch all chunks */ - table_relation_fetch_toast_slice(toastrel, toast_pointer.va_valueid, + table_relation_fetch_toast_slice(toastrel, valueid, attrsize, 0, attrsize, result); /* Close toast table */ @@ -398,23 +433,43 @@ toast_fetch_datum_slice(varlena *attr, int32 sliceoffset, { Relation toastrel; varlena *result; - varatt_external_oid toast_pointer; int32 attrsize; + Oid toastrelid; + Oid8 valueid; + bool is_compressed; if (!VARATT_IS_EXTERNAL_ONDISK(attr)) elog(ERROR, "toast_fetch_datum_slice shouldn't be called for non-ondisk datums"); - /* Must copy to access aligned fields */ - VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + if (VARTAG_EXTERNAL(attr) == VARTAG_ONDISK_OID8) + { + varatt_external_oid8 toast_pointer; + + /* Must copy to access aligned fields */ + VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + attrsize = VARATT_EXTERNAL_OID8_GET_EXTSIZE(toast_pointer); + toastrelid = toast_pointer.va_toastrelid; + valueid = VARATT_EXTERNAL_OID8_GET_VALUEID(toast_pointer); + is_compressed = VARATT_EXTERNAL_OID8_IS_COMPRESSED(toast_pointer); + } + else + { + varatt_external_oid toast_pointer; + + /* Must copy to access aligned fields */ + VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + attrsize = VARATT_EXTERNAL_OID_GET_EXTSIZE(toast_pointer); + toastrelid = toast_pointer.va_toastrelid; + valueid = toast_pointer.va_valueid; + is_compressed = VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer); + } /* * It's nonsense to fetch slices of a compressed datum unless when it's a * prefix -- this isn't lo_* we can't return a compressed datum which is * meaningful to toast later. */ - Assert(!VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer) || 0 == sliceoffset); - - attrsize = VARATT_EXTERNAL_OID_GET_EXTSIZE(toast_pointer); + Assert(!is_compressed || 0 == sliceoffset); if (sliceoffset >= attrsize) { @@ -427,7 +482,7 @@ toast_fetch_datum_slice(varlena *attr, int32 sliceoffset, * space required by va_tcinfo, which is stored at the beginning as an * int32 value. */ - if (VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer) && slicelength > 0) + if (is_compressed && slicelength > 0) slicelength = slicelength + sizeof(int32); /* @@ -440,7 +495,7 @@ toast_fetch_datum_slice(varlena *attr, int32 sliceoffset, result = (varlena *) palloc(slicelength + VARHDRSZ); - if (VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer)) + if (is_compressed) SET_VARSIZE_COMPRESSED(result, slicelength + VARHDRSZ); else SET_VARSIZE(result, slicelength + VARHDRSZ); @@ -449,10 +504,10 @@ toast_fetch_datum_slice(varlena *attr, int32 sliceoffset, return result; /* Can save a lot of work at this point! */ /* Open the toast relation */ - toastrel = table_open(toast_pointer.va_toastrelid, AccessShareLock); + toastrel = table_open(toastrelid, AccessShareLock); /* Fetch all chunks */ - table_relation_fetch_toast_slice(toastrel, toast_pointer.va_valueid, + table_relation_fetch_toast_slice(toastrel, valueid, attrsize, sliceoffset, slicelength, result); @@ -550,10 +605,20 @@ toast_raw_datum_size(Datum value) if (VARATT_IS_EXTERNAL_ONDISK(attr)) { /* va_rawsize is the size of the original datum -- including header */ - varatt_external_oid toast_pointer; + if (VARTAG_EXTERNAL(attr) == VARTAG_ONDISK_OID8) + { + varatt_external_oid8 toast_pointer; - VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); - result = toast_pointer.va_rawsize; + VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + result = toast_pointer.va_rawsize; + } + else + { + varatt_external_oid toast_pointer; + + VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + result = toast_pointer.va_rawsize; + } } else if (VARATT_IS_EXTERNAL_INDIRECT(attr)) { @@ -610,10 +675,20 @@ toast_datum_size(Datum value) * compressed or not. We do not count the size of the toast pointer * ... should we? */ - varatt_external_oid toast_pointer; + if (VARTAG_EXTERNAL(attr) == VARTAG_ONDISK_OID8) + { + varatt_external_oid8 toast_pointer; - VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); - result = VARATT_EXTERNAL_OID_GET_EXTSIZE(toast_pointer); + VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + result = VARATT_EXTERNAL_OID8_GET_EXTSIZE(toast_pointer); + } + else + { + varatt_external_oid toast_pointer; + + VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + result = VARATT_EXTERNAL_OID_GET_EXTSIZE(toast_pointer); + } } else if (VARATT_IS_EXTERNAL_INDIRECT(attr)) { diff --git a/src/backend/access/common/reloptions.c b/src/backend/access/common/reloptions.c index dfb28e592915..ea9a04179090 100644 --- a/src/backend/access/common/reloptions.c +++ b/src/backend/access/common/reloptions.c @@ -579,7 +579,7 @@ static relopt_enum enumRelOpts[] = }, StdRdOptToastValueTypes, STDRD_OPTION_TOAST_VALUE_TYPE_OID, - gettext_noop("Valid values are \"oid\".") + gettext_noop("Valid values are \"oid\" and \"oid8\".") }, { { diff --git a/src/backend/access/common/toast_compression.c b/src/backend/access/common/toast_compression.c index 3b1bd6519261..53b31943cdfe 100644 --- a/src/backend/access/common/toast_compression.c +++ b/src/backend/access/common/toast_compression.c @@ -260,7 +260,16 @@ toast_get_compression_id(varlena *attr) * the external toast pointer. If compressed inline, fetch it from the * toast compression header. */ - if (VARATT_IS_EXTERNAL_ONDISK(attr)) + if (VARATT_IS_EXTERNAL_ONDISK_OID8(attr)) + { + varatt_external_oid8 toast_pointer; + + VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + + if (VARATT_EXTERNAL_OID8_IS_COMPRESSED(toast_pointer)) + cmid = VARATT_EXTERNAL_OID8_GET_COMPRESS_METHOD(toast_pointer); + } + else if (VARATT_IS_EXTERNAL_ONDISK(attr)) { varatt_external_oid toast_pointer; diff --git a/src/backend/access/common/toast_internals.c b/src/backend/access/common/toast_internals.c index 2c180491c2ff..7b37798e1c08 100644 --- a/src/backend/access/common/toast_internals.c +++ b/src/backend/access/common/toast_internals.c @@ -125,7 +125,6 @@ toast_save_datum(Relation rel, Datum value, TupleDesc toasttupDesc; CommandId mycid = GetCurrentCommandId(true); varlena *result; - varatt_external_oid toast_pointer; int32 chunk_seq = 0; char *data_p; int32 data_todo; @@ -133,9 +132,17 @@ toast_save_datum(Relation rel, Datum value, int num_indexes; int validIndex; Oid toast_typid = get_atttype(rel->rd_rel->reltoastrelid, 1); + int32 max_chunk_size; + + /* Fields that will be assembled into the TOAST pointer at the end */ + int32 va_rawsize; + uint32 va_extinfo; + Oid8 va_valueid; + Oid va_toastrelid; Assert(!VARATT_IS_EXTERNAL(dval)); Assert(OidIsValid(toast_typid)); + Assert(toast_typid == OIDOID || toast_typid == OID8OID); /* * Open the toast relation and its indexes. We can use the index to check @@ -165,28 +172,32 @@ toast_save_datum(Relation rel, Datum value, { data_p = VARDATA_SHORT(dval); data_todo = VARSIZE_SHORT(dval) - VARHDRSZ_SHORT; - toast_pointer.va_rawsize = data_todo + VARHDRSZ; /* as if not short */ - toast_pointer.va_extinfo = data_todo; + va_rawsize = data_todo + VARHDRSZ; /* as if not short */ + va_extinfo = data_todo; } else if (VARATT_IS_COMPRESSED(dval)) { + uint32 cmid = VARDATA_COMPRESSED_GET_COMPRESS_METHOD(dval); + data_p = VARDATA(dval); data_todo = VARSIZE(dval) - VARHDRSZ; /* rawsize in a compressed datum is just the size of the payload */ - toast_pointer.va_rawsize = VARDATA_COMPRESSED_GET_EXTSIZE(dval) + VARHDRSZ; + va_rawsize = VARDATA_COMPRESSED_GET_EXTSIZE(dval) + VARHDRSZ; /* set external size and compression method */ - VARATT_EXTERNAL_SET_SIZE_AND_COMPRESS_METHOD(toast_pointer, data_todo, - VARDATA_COMPRESSED_GET_COMPRESS_METHOD(dval)); + Assert(cmid == TOAST_PGLZ_COMPRESSION_ID || + cmid == TOAST_LZ4_COMPRESSION_ID); + va_extinfo = data_todo | (cmid << VARLENA_EXTSIZE_BITS); + /* Assert that the numbers look like it's compressed */ - Assert(VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer)); + Assert((va_extinfo & VARLENA_EXTSIZE_MASK) < (Size) (va_rawsize - VARHDRSZ)); } else { data_p = VARDATA(dval); data_todo = VARSIZE(dval) - VARHDRSZ; - toast_pointer.va_rawsize = VARSIZE(dval); - toast_pointer.va_extinfo = data_todo; + va_rawsize = VARSIZE(dval); + va_extinfo = data_todo; } /* @@ -198,13 +209,14 @@ toast_save_datum(Relation rel, Datum value, * if we have to substitute such an OID. */ if (OidIsValid(rel->rd_toastoid)) - toast_pointer.va_toastrelid = rel->rd_toastoid; + va_toastrelid = rel->rd_toastoid; else - toast_pointer.va_toastrelid = RelationGetRelid(toastrel); + va_toastrelid = RelationGetRelid(toastrel); /* - * Choose a new value to use as the value ID for this toast value, be it - * for OID or OID8 TOAST relations. + * Choose a new value to use as the value ID for this toast value, and + * determine the maximum chunk size based on the TOAST table's chunk_id + * type. * * Normally we just choose an unused value within the toast table. But * during table-rewriting operations where we are preserving an existing @@ -213,84 +225,123 @@ toast_save_datum(Relation rel, Datum value, * toast table, re-use its value ID. If we didn't have a prior external * value (which is a corner case, but possible if the table's attstorage * options have been changed), we have to pick a value ID that doesn't - * conflict with either new or existing toast value IDs. If the TOAST - * table uses 8-byte value IDs, we should not really care much about - * that. + * conflict with either new or existing toast value IDs. For Oid8 tables, + * value conflicts are not a concern. */ - if (!OidIsValid(rel->rd_toastoid)) + if (toast_typid == OID8OID) { - /* normal case: just choose an unused OID */ - if (toast_typid == OIDOID) - toast_pointer.va_valueid = - GetNewOidWithIndex(toastrel, - RelationGetRelid(toastidxs[validIndex]), - (AttrNumber) 1); - else if (toast_typid == OID8OID) - toast_pointer.va_valueid = GetNewObjectId8(); + if (!OidIsValid(rel->rd_toastoid)) + { + /* normal case: just choose a new Oid8 */ + va_valueid = GetNewObjectId8(); + } else - Assert(false); + { + /* rewrite case: check to see if value was in old toast table */ + va_valueid = InvalidOid8; + if (oldexternal != NULL) + { + varatt_external_oid8 old_toast_pointer; + + Assert(VARATT_IS_EXTERNAL_ONDISK(oldexternal)); + /* Must copy to access aligned fields */ + VARATT_EXTERNAL_GET_POINTER(old_toast_pointer, oldexternal); + if (old_toast_pointer.va_toastrelid == rel->rd_toastoid) + { + /* This value came from the old toast table; reuse its ID */ + va_valueid = VARATT_EXTERNAL_OID8_GET_VALUEID(old_toast_pointer); + /* + * There is a corner case here: the table rewrite might + * have to copy both live and recently-dead versions of a + * row, and those versions could easily reference the same + * toast value. When we copy the second or later version + * of such a row, reusing the OID will mean we select an + * OID that's already in the new toast table. Check for + * that, and if so, just fall through without writing the + * data again. + * + * While annoying and ugly-looking, this is a good thing + * because it ensures that we wind up with only one copy + * of the toast value when there is only one copy in the + * old toast table. Before we detected this case, we'd + * have made multiple copies, wasting space; and what's + * worse, the copies belonging to already-deleted heap + * tuples would not be reclaimed by VACUUM. + */ + if (toastrel_valueid_exists(toastrel, va_valueid)) + { + /* Match, so short-circuit the data storage loop below */ + data_todo = 0; + } + } + } + if (va_valueid == InvalidOid8) + { + /* + * New value. For Oid8 we just pick a new ID without worrying + * about conflicts. + */ + va_valueid = GetNewObjectId8(); + } + } + + max_chunk_size = TOAST_OID8_MAX_CHUNK_SIZE; } else { - /* rewrite case: check to see if value was in old toast table */ - toast_pointer.va_valueid = InvalidOid; - if (oldexternal != NULL) + if (!OidIsValid(rel->rd_toastoid)) { - varatt_external_oid old_toast_pointer; - - Assert(VARATT_IS_EXTERNAL_ONDISK(oldexternal)); - /* Must copy to access aligned fields */ - VARATT_EXTERNAL_GET_POINTER(old_toast_pointer, oldexternal); - if (old_toast_pointer.va_toastrelid == rel->rd_toastoid) + /* normal case: just choose an unused OID */ + va_valueid = + GetNewOidWithIndex(toastrel, + RelationGetRelid(toastidxs[validIndex]), + (AttrNumber) 1); + } + else + { + /* rewrite case: check to see if value was in old toast table */ + va_valueid = InvalidOid; + if (oldexternal != NULL) { - /* This value came from the old toast table; reuse its OID */ - toast_pointer.va_valueid = old_toast_pointer.va_valueid; + varatt_external_oid old_toast_pointer; - /* - * There is a corner case here: the table rewrite might have - * to copy both live and recently-dead versions of a row, and - * those versions could easily reference the same toast value. - * When we copy the second or later version of such a row, - * reusing the OID will mean we select an OID that's already - * in the new toast table. Check for that, and if so, just - * fall through without writing the data again. - * - * While annoying and ugly-looking, this is a good thing - * because it ensures that we wind up with only one copy of - * the toast value when there is only one copy in the old - * toast table. Before we detected this case, we'd have made - * multiple copies, wasting space; and what's worse, the - * copies belonging to already-deleted heap tuples would not - * be reclaimed by VACUUM. - */ - if (toastrel_valueid_exists(toastrel, - toast_pointer.va_valueid)) + Assert(VARATT_IS_EXTERNAL_ONDISK(oldexternal)); + /* Must copy to access aligned fields */ + VARATT_EXTERNAL_GET_POINTER(old_toast_pointer, oldexternal); + if (old_toast_pointer.va_toastrelid == rel->rd_toastoid) { - /* Match, so short-circuit the data storage loop below */ - data_todo = 0; + /* This value came from the old toast table; reuse its OID */ + va_valueid = old_toast_pointer.va_valueid; + + /* + * Corner case during table rewrite with multiple + * versions of the same row. See above for details. + */ + if (toastrel_valueid_exists(toastrel, va_valueid)) + { + /* Match, so short-circuit the data storage loop below */ + data_todo = 0; + } } } - } - if (toast_pointer.va_valueid == InvalidOid) - { - /* - * new value; must choose a value that doesn't conflict in either - * old or new toast table. - */ - if (toast_typid == OIDOID) + if (va_valueid == InvalidOid) { + /* + * new value; must choose a value that doesn't conflict in + * either old or new toast table. + */ do { - toast_pointer.va_valueid = + va_valueid = GetNewOidWithIndex(toastrel, RelationGetRelid(toastidxs[validIndex]), (AttrNumber) 1); } while (toastid_valueid_exists(rel->rd_toastoid, - toast_pointer.va_valueid)); + va_valueid)); } - else if (toast_typid == OID8OID) - toast_pointer.va_valueid = GetNewObjectId8(); } + + max_chunk_size = TOAST_OID_MAX_CHUNK_SIZE; } /* @@ -314,15 +365,15 @@ toast_save_datum(Relation rel, Datum value, /* * Calculate the size of this chunk */ - chunk_size = Min(TOAST_OID_MAX_CHUNK_SIZE, data_todo); + chunk_size = Min(max_chunk_size, data_todo); /* * Build a tuple and store it */ - if (toast_typid == OIDOID) - t_values[0] = ObjectIdGetDatum(toast_pointer.va_valueid); - else if (toast_typid == OID8OID) - t_values[0] = ObjectId8GetDatum(toast_pointer.va_valueid); + if (toast_typid == OID8OID) + t_values[0] = ObjectId8GetDatum(va_valueid); + else + t_values[0] = ObjectIdGetDatum((Oid) va_valueid); t_values[1] = Int32GetDatum(chunk_seq++); SET_VARSIZE(&chunk_data, chunk_size + VARHDRSZ); memcpy(VARDATA(&chunk_data), data_p, chunk_size); @@ -376,11 +427,34 @@ toast_save_datum(Relation rel, Datum value, table_close(toastrel, NoLock); /* - * Create the TOAST pointer value that we'll return + * Create the TOAST pointer value that we'll return. */ - result = (varlena *) palloc(TOAST_OID_POINTER_SIZE); - SET_VARTAG_EXTERNAL(result, VARTAG_ONDISK_OID); - memcpy(VARDATA_EXTERNAL(result), &toast_pointer, sizeof(toast_pointer)); + if (toast_typid == OID8OID) + { + varatt_external_oid8 toast_pointer; + + toast_pointer.va_rawsize = va_rawsize; + toast_pointer.va_extinfo = va_extinfo; + VARATT_EXTERNAL_OID8_SET_VALUEID(toast_pointer, va_valueid); + toast_pointer.va_toastrelid = va_toastrelid; + + result = (varlena *) palloc(TOAST_OID8_POINTER_SIZE); + SET_VARTAG_EXTERNAL(result, VARTAG_ONDISK_OID8); + memcpy(VARDATA_EXTERNAL(result), &toast_pointer, sizeof(toast_pointer)); + } + else + { + varatt_external_oid toast_pointer; + + toast_pointer.va_rawsize = va_rawsize; + toast_pointer.va_extinfo = va_extinfo; + toast_pointer.va_valueid = (Oid) va_valueid; + toast_pointer.va_toastrelid = va_toastrelid; + + result = (varlena *) palloc(TOAST_OID_POINTER_SIZE); + SET_VARTAG_EXTERNAL(result, VARTAG_ONDISK_OID); + memcpy(VARDATA_EXTERNAL(result), &toast_pointer, sizeof(toast_pointer)); + } return PointerGetDatum(result); } @@ -395,7 +469,6 @@ void toast_delete_datum(Relation rel, Datum value, bool is_speculative) { varlena *attr = (varlena *) DatumGetPointer(value); - varatt_external_oid toast_pointer; Relation toastrel; Relation *toastidxs; ScanKeyData toastkey; @@ -403,42 +476,75 @@ toast_delete_datum(Relation rel, Datum value, bool is_speculative) HeapTuple toasttup; int num_indexes; int validIndex; - Oid toast_typid; + Oid toastrelid; + vartag_external tag; if (!VARATT_IS_EXTERNAL_ONDISK(attr)) return; - /* Must copy to access aligned fields */ - VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); - /* - * Open the toast relation and its indexes + * Determine the pointer type from the datum's vartag and extract the + * toast relation OID and value ID accordingly. The vartag tells us + * everything we need - no TOAST table schema lookup required. */ - toastrel = table_open(toast_pointer.va_toastrelid, RowExclusiveLock); - toast_typid = TupleDescAttr(toastrel->rd_att, 0)->atttypid; - Assert(toast_typid == OIDOID || toast_typid == OID8OID); + tag = VARTAG_EXTERNAL(attr); - /* Fetch valid relation used for process */ - validIndex = toast_open_indexes(toastrel, - RowExclusiveLock, - &toastidxs, - &num_indexes); + if (tag == VARTAG_ONDISK_OID8) + { + varatt_external_oid8 toast_pointer8; - /* - * Setup a scan key to find chunks with matching va_valueid - */ - if (toast_typid == OIDOID) + /* Must copy to access aligned fields */ + VARATT_EXTERNAL_GET_POINTER(toast_pointer8, attr); + toastrelid = toast_pointer8.va_toastrelid; + + /* + * Open the toast relation and its indexes + */ + toastrel = table_open(toastrelid, RowExclusiveLock); + + /* Fetch valid relation used for process */ + validIndex = toast_open_indexes(toastrel, + RowExclusiveLock, + &toastidxs, + &num_indexes); + + /* + * Setup a scan key to find chunks with matching va_valueid + */ + ScanKeyInit(&toastkey, + (AttrNumber) 1, + BTEqualStrategyNumber, F_OID8EQ, + ObjectId8GetDatum(VARATT_EXTERNAL_OID8_GET_VALUEID(toast_pointer8))); + } + else + { + varatt_external_oid toast_pointer; + + Assert(tag == VARTAG_ONDISK_OID); + + /* Must copy to access aligned fields */ + VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + toastrelid = toast_pointer.va_toastrelid; + + /* + * Open the toast relation and its indexes + */ + toastrel = table_open(toastrelid, RowExclusiveLock); + + /* Fetch valid relation used for process */ + validIndex = toast_open_indexes(toastrel, + RowExclusiveLock, + &toastidxs, + &num_indexes); + + /* + * Setup a scan key to find chunks with matching va_valueid + */ ScanKeyInit(&toastkey, (AttrNumber) 1, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(toast_pointer.va_valueid)); - else if (toast_typid == OID8OID) - ScanKeyInit(&toastkey, - (AttrNumber) 1, - BTEqualStrategyNumber, F_OID8EQ, - ObjectId8GetDatum(toast_pointer.va_valueid)); - else - Assert(false); + } /* * Find all the chunks. (We don't actually care whether we see them in diff --git a/src/backend/access/heap/heaptoast.c b/src/backend/access/heap/heaptoast.c index a93e98788da7..364aec11fc88 100644 --- a/src/backend/access/heap/heaptoast.c +++ b/src/backend/access/heap/heaptoast.c @@ -650,7 +650,10 @@ heap_fetch_toast_slice(Relation toastrel, Oid8 valueid, int32 attrsize, toast_typid = TupleDescAttr(toastrel->rd_att, 0)->atttypid; Assert(toast_typid == OIDOID || toast_typid == OID8OID); - max_chunk_size = TOAST_OID_MAX_CHUNK_SIZE; + if (toast_typid == OID8OID) + max_chunk_size = TOAST_OID8_MAX_CHUNK_SIZE; + else + max_chunk_size = TOAST_OID_MAX_CHUNK_SIZE; totalchunks = ((attrsize - 1) / max_chunk_size) + 1; startchunk = sliceoffset / max_chunk_size; diff --git a/src/backend/access/table/toast_helper.c b/src/backend/access/table/toast_helper.c index 6f2d691cc681..6c7257f5f94b 100644 --- a/src/backend/access/table/toast_helper.c +++ b/src/backend/access/table/toast_helper.c @@ -69,12 +69,16 @@ toast_tuple_init(ToastTupleContext *ttc) /* * If the old value is stored on disk, check if it has changed so * we have to delete it later. + * + * Note that TOAST pointers could have different vartags, for oid + * or oid8, and these can have a different sizes. */ if (att->attlen == -1 && !ttc->ttc_oldisnull[i] && VARATT_IS_EXTERNAL_ONDISK(old_value)) { if (ttc->ttc_isnull[i] || !VARATT_IS_EXTERNAL_ONDISK(new_value) || + VARTAG_EXTERNAL(old_value) != VARTAG_EXTERNAL(new_value) || memcmp(old_value, new_value, VARSIZE_EXTERNAL(old_value)) != 0) { diff --git a/src/backend/replication/logical/reorderbuffer.c b/src/backend/replication/logical/reorderbuffer.c index dd4b6e8f7f45..644e1415cec2 100644 --- a/src/backend/replication/logical/reorderbuffer.c +++ b/src/backend/replication/logical/reorderbuffer.c @@ -5176,12 +5176,14 @@ ReorderBufferToastReplace(ReorderBuffer *rb, ReorderBufferTXN *txn, /* va_rawsize is the size of the original datum -- including header */ varatt_external_oid toast_pointer; + varatt_external_oid8 toast_pointer8; varatt_indirect redirect_pointer; varlena *new_datum = NULL; varlena *reconstructed; dlist_iter it; Size data_done = 0; Oid8 toast_valueid; + int32 rawsize; if (attr->attisdropped) continue; @@ -5201,8 +5203,19 @@ ReorderBufferToastReplace(ReorderBuffer *rb, ReorderBufferTXN *txn, if (!VARATT_IS_EXTERNAL(varlena_pointer)) continue; - VARATT_EXTERNAL_GET_POINTER(toast_pointer, varlena_pointer); - toast_valueid = toast_pointer.va_valueid; + /* Branch on vartag to handle both pointer types */ + if (VARTAG_EXTERNAL(varlena_pointer) == VARTAG_ONDISK_OID8) + { + VARATT_EXTERNAL_GET_POINTER(toast_pointer8, varlena_pointer); + toast_valueid = VARATT_EXTERNAL_OID8_GET_VALUEID(toast_pointer8); + rawsize = toast_pointer8.va_rawsize; + } + else + { + VARATT_EXTERNAL_GET_POINTER(toast_pointer, varlena_pointer); + toast_valueid = toast_pointer.va_valueid; + rawsize = toast_pointer.va_rawsize; + } /* * Check whether the toast tuple changed, replace if so. @@ -5220,7 +5233,7 @@ ReorderBufferToastReplace(ReorderBuffer *rb, ReorderBufferTXN *txn, free[natt] = true; - reconstructed = palloc0(toast_pointer.va_rawsize); + reconstructed = palloc0(rawsize); ent->reconstructed = reconstructed; @@ -5245,13 +5258,28 @@ ReorderBufferToastReplace(ReorderBuffer *rb, ReorderBufferTXN *txn, VARSIZE(chunk) - VARHDRSZ); data_done += VARSIZE(chunk) - VARHDRSZ; } - Assert(data_done == VARATT_EXTERNAL_OID_GET_EXTSIZE(toast_pointer)); - /* make sure its marked as compressed or not */ - if (VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer)) - SET_VARSIZE_COMPRESSED(reconstructed, data_done + VARHDRSZ); + /* Verify size and set compression status based on pointer type */ + if (VARTAG_EXTERNAL(varlena_pointer) == VARTAG_ONDISK_OID8) + { + Assert(data_done == VARATT_EXTERNAL_OID8_GET_EXTSIZE(toast_pointer8)); + + /* make sure its marked as compressed or not */ + if (VARATT_EXTERNAL_OID8_IS_COMPRESSED(toast_pointer8)) + SET_VARSIZE_COMPRESSED(reconstructed, data_done + VARHDRSZ); + else + SET_VARSIZE(reconstructed, data_done + VARHDRSZ); + } else - SET_VARSIZE(reconstructed, data_done + VARHDRSZ); + { + Assert(data_done == VARATT_EXTERNAL_OID_GET_EXTSIZE(toast_pointer)); + + /* make sure its marked as compressed or not */ + if (VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer)) + SET_VARSIZE_COMPRESSED(reconstructed, data_done + VARHDRSZ); + else + SET_VARSIZE(reconstructed, data_done + VARHDRSZ); + } memset(&redirect_pointer, 0, sizeof(redirect_pointer)); redirect_pointer.pointer = reconstructed; diff --git a/src/backend/utils/adt/varlena.c b/src/backend/utils/adt/varlena.c index 1e264904a6e1..e104a6308d65 100644 --- a/src/backend/utils/adt/varlena.c +++ b/src/backend/utils/adt/varlena.c @@ -4261,7 +4261,7 @@ pg_column_toast_chunk_id(PG_FUNCTION_ARGS) { int typlen; varlena *attr; - varatt_external_oid toast_pointer; + Oid8 result; /* On first call, get the input type's typlen, and save at *fn_extra */ if (fcinfo->flinfo->fn_extra == NULL) @@ -4288,9 +4288,22 @@ pg_column_toast_chunk_id(PG_FUNCTION_ARGS) if (!VARATT_IS_EXTERNAL_ONDISK(attr)) PG_RETURN_NULL(); - VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + if (VARATT_IS_EXTERNAL_ONDISK_OID8(attr)) + { + varatt_external_oid8 toast_pointer; - PG_RETURN_OID8(toast_pointer.va_valueid); + VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + result = VARATT_EXTERNAL_OID8_GET_VALUEID(toast_pointer); + } + else + { + varatt_external_oid toast_pointer; + + VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + result = toast_pointer.va_valueid; + } + + PG_RETURN_OID8(result); } /* diff --git a/src/test/regress/expected/compression.out b/src/test/regress/expected/compression.out index 09f198149aa4..e9128193515a 100644 --- a/src/test/regress/expected/compression.out +++ b/src/test/regress/expected/compression.out @@ -66,6 +66,47 @@ SELECT SUBSTR(f1, 200, 5) FROM cmdata2; (1 row) DROP TABLE cmdata2; +-- pg_column_compression() with oid and oid8 +CREATE TABLE toastcomp_oid(f1 text) WITH (toast_value_type = 'oid'); +CREATE TABLE toastcomp_oid8(f1 text) WITH (toast_value_type = 'oid8'); +ALTER TABLE toastcomp_oid ALTER COLUMN f1 SET STORAGE EXTERNAL; +ALTER TABLE toastcomp_oid8 ALTER COLUMN f1 SET STORAGE EXTERNAL; +INSERT INTO toastcomp_oid VALUES (repeat('1234567890', 10000)); +INSERT INTO toastcomp_oid8 VALUES (repeat('1234567890', 10000)); +SELECT pg_column_compression(f1) IS NULL AS uncompressed FROM toastcomp_oid; + uncompressed +-------------- + t +(1 row) + +SELECT pg_column_compression(f1) IS NULL AS uncompressed FROM toastcomp_oid8; + uncompressed +-------------- + t +(1 row) + +-- out-of-line and compressed. +TRUNCATE toastcomp_oid; +TRUNCATE toastcomp_oid8; +ALTER TABLE toastcomp_oid ALTER COLUMN f1 SET STORAGE EXTENDED; +ALTER TABLE toastcomp_oid8 ALTER COLUMN f1 SET STORAGE EXTENDED; +ALTER TABLE toastcomp_oid SET (toast_tuple_target = 128); +ALTER TABLE toastcomp_oid8 SET (toast_tuple_target = 128); +INSERT INTO toastcomp_oid VALUES (repeat('1234567890', 10000)); +INSERT INTO toastcomp_oid8 VALUES (repeat('1234567890', 10000)); +SELECT pg_column_compression(f1) FROM toastcomp_oid; + pg_column_compression +----------------------- + pglz +(1 row) + +SELECT pg_column_compression(f1) FROM toastcomp_oid8; + pg_column_compression +----------------------- + pglz +(1 row) + +DROP TABLE toastcomp_oid, toastcomp_oid8; --test column type update varlena/non-varlena CREATE TABLE cmdata2 (f1 int); \d+ cmdata2 diff --git a/src/test/regress/expected/strings.out b/src/test/regress/expected/strings.out index 2e46519df8cd..ba1961baa155 100644 --- a/src/test/regress/expected/strings.out +++ b/src/test/regress/expected/strings.out @@ -2286,6 +2286,49 @@ SELECT c1.relname, a.atttypid::regtype ALTER TABLE toasttest_oid RESET (toast_value_type); ALTER TABLE toasttest_oid8 RESET (toast_value_type); +-- UPDATE with out-of-line datum that belongs to another TOAST table. +CREATE TABLE toastupd_oid(f1 text) WITH (toast_value_type = 'oid'); +CREATE TABLE toastupd_oid8(f1 text) WITH (toast_value_type = 'oid8'); +ALTER TABLE toastupd_oid ALTER COLUMN f1 SET STORAGE EXTERNAL; +ALTER TABLE toastupd_oid8 ALTER COLUMN f1 SET STORAGE EXTERNAL; +SELECT reltoastrelid::regclass AS upd_oid_toast FROM pg_class + WHERE oid = 'toastupd_oid'::regclass \gset +SELECT reltoastrelid::regclass AS upd_oid8_toast FROM pg_class + WHERE oid = 'toastupd_oid8'::regclass \gset +INSERT INTO toastupd_oid VALUES (repeat('a', 100000)); +INSERT INTO toastupd_oid8 VALUES (repeat('b', 100000)); +-- old value is an oid8 pointer, new value an oid pointer. +UPDATE toastupd_oid8 SET f1 = toastupd_oid.f1 FROM toastupd_oid; +SELECT length(f1), substr(f1, 1, 3) FROM toastupd_oid8; + length | substr +--------+-------- + 100000 | aaa +(1 row) + +-- replaced value must be gone, leaving a single value behind. +SELECT count(*) FROM :upd_oid8_toast WHERE chunk_seq = 0; + count +------- + 1 +(1 row) + +-- reverse: old value is an oid pointer, new value an oid8 pointer. +TRUNCATE toastupd_oid; +INSERT INTO toastupd_oid VALUES (repeat('c', 100000)); +UPDATE toastupd_oid SET f1 = toastupd_oid8.f1 FROM toastupd_oid8; +SELECT length(f1), substr(f1, 1, 3) FROM toastupd_oid; + length | substr +--------+-------- + 100000 | aaa +(1 row) + +SELECT count(*) FROM :upd_oid_toast WHERE chunk_seq = 0; + count +------- + 1 +(1 row) + +DROP TABLE toastupd_oid, toastupd_oid8; -- test internally compressing datums -- this tests compressing a datum to a very small size which exercises a -- corner case in packed-varlena handling: even though small, the compressed diff --git a/src/test/regress/sql/compression.sql b/src/test/regress/sql/compression.sql index ce5ea37a660c..229143f06ddd 100644 --- a/src/test/regress/sql/compression.sql +++ b/src/test/regress/sql/compression.sql @@ -37,6 +37,28 @@ SELECT pg_column_compression(f1) FROM cmdata2; SELECT SUBSTR(f1, 200, 5) FROM cmdata2; DROP TABLE cmdata2; +-- pg_column_compression() with oid and oid8 +CREATE TABLE toastcomp_oid(f1 text) WITH (toast_value_type = 'oid'); +CREATE TABLE toastcomp_oid8(f1 text) WITH (toast_value_type = 'oid8'); +ALTER TABLE toastcomp_oid ALTER COLUMN f1 SET STORAGE EXTERNAL; +ALTER TABLE toastcomp_oid8 ALTER COLUMN f1 SET STORAGE EXTERNAL; +INSERT INTO toastcomp_oid VALUES (repeat('1234567890', 10000)); +INSERT INTO toastcomp_oid8 VALUES (repeat('1234567890', 10000)); +SELECT pg_column_compression(f1) IS NULL AS uncompressed FROM toastcomp_oid; +SELECT pg_column_compression(f1) IS NULL AS uncompressed FROM toastcomp_oid8; +-- out-of-line and compressed. +TRUNCATE toastcomp_oid; +TRUNCATE toastcomp_oid8; +ALTER TABLE toastcomp_oid ALTER COLUMN f1 SET STORAGE EXTENDED; +ALTER TABLE toastcomp_oid8 ALTER COLUMN f1 SET STORAGE EXTENDED; +ALTER TABLE toastcomp_oid SET (toast_tuple_target = 128); +ALTER TABLE toastcomp_oid8 SET (toast_tuple_target = 128); +INSERT INTO toastcomp_oid VALUES (repeat('1234567890', 10000)); +INSERT INTO toastcomp_oid8 VALUES (repeat('1234567890', 10000)); +SELECT pg_column_compression(f1) FROM toastcomp_oid; +SELECT pg_column_compression(f1) FROM toastcomp_oid8; +DROP TABLE toastcomp_oid, toastcomp_oid8; + --test column type update varlena/non-varlena CREATE TABLE cmdata2 (f1 int); \d+ cmdata2 diff --git a/src/test/regress/sql/strings.sql b/src/test/regress/sql/strings.sql index 5e260ddaefc1..5c41fb2d1ea5 100644 --- a/src/test/regress/sql/strings.sql +++ b/src/test/regress/sql/strings.sql @@ -723,6 +723,31 @@ SELECT c1.relname, a.atttypid::regtype ALTER TABLE toasttest_oid RESET (toast_value_type); ALTER TABLE toasttest_oid8 RESET (toast_value_type); +-- UPDATE with out-of-line datum that belongs to another TOAST table. +CREATE TABLE toastupd_oid(f1 text) WITH (toast_value_type = 'oid'); +CREATE TABLE toastupd_oid8(f1 text) WITH (toast_value_type = 'oid8'); +ALTER TABLE toastupd_oid ALTER COLUMN f1 SET STORAGE EXTERNAL; +ALTER TABLE toastupd_oid8 ALTER COLUMN f1 SET STORAGE EXTERNAL; +SELECT reltoastrelid::regclass AS upd_oid_toast FROM pg_class + WHERE oid = 'toastupd_oid'::regclass \gset +SELECT reltoastrelid::regclass AS upd_oid8_toast FROM pg_class + WHERE oid = 'toastupd_oid8'::regclass \gset +INSERT INTO toastupd_oid VALUES (repeat('a', 100000)); +INSERT INTO toastupd_oid8 VALUES (repeat('b', 100000)); +-- old value is an oid8 pointer, new value an oid pointer. +UPDATE toastupd_oid8 SET f1 = toastupd_oid.f1 FROM toastupd_oid; +SELECT length(f1), substr(f1, 1, 3) FROM toastupd_oid8; +-- replaced value must be gone, leaving a single value behind. +SELECT count(*) FROM :upd_oid8_toast WHERE chunk_seq = 0; +-- reverse: old value is an oid pointer, new value an oid8 pointer. +TRUNCATE toastupd_oid; +INSERT INTO toastupd_oid VALUES (repeat('c', 100000)); +UPDATE toastupd_oid SET f1 = toastupd_oid8.f1 FROM toastupd_oid8; +SELECT length(f1), substr(f1, 1, 3) FROM toastupd_oid; +SELECT count(*) FROM :upd_oid_toast WHERE chunk_seq = 0; + +DROP TABLE toastupd_oid, toastupd_oid8; + -- test internally compressing datums -- this tests compressing a datum to a very small size which exercises a diff --git a/doc/src/sgml/storage.sgml b/doc/src/sgml/storage.sgml index 21bd67c11ea0..3561a6a67ece 100644 --- a/doc/src/sgml/storage.sgml +++ b/doc/src/sgml/storage.sgml @@ -417,7 +417,9 @@ described in more detail below. Out-of-line values are divided (after compression if used) into chunks of at -most TOAST_OID_MAX_CHUNK_SIZE bytes (by default this value is chosen +most TOAST_OID_MAX_CHUNK_SIZE or +TOAST_OID8_MAX_CHUNK_SIZE bytes depending on the +chunk_id type (by default this value is chosen so that four chunk rows will fit on a page, making it about 2000 bytes). Each chunk is stored as a separate row in the TOAST table belonging to the owning table. Every @@ -434,8 +436,10 @@ retrieval of the values. A pointer datum representing an out-of-line on-disk logical datum size (original uncompressed data length), physical stored size (different if compression was applied), and the compression method used, if any. Allowing for the varlena header bytes, -the total size of an on-disk TOAST pointer datum is therefore 18 -bytes regardless of the actual size of the represented value. +the total size of an on-disk TOAST pointer datum is 18 +bytes when using an OID as chunk_id, or 22 bytes +when using an OID8 as chunk_id, regardless of the +actual size of the represented value. diff --git a/contrib/amcheck/verify_heapam.c b/contrib/amcheck/verify_heapam.c index 42f409548e1d..ef631408b8ef 100644 --- a/contrib/amcheck/verify_heapam.c +++ b/contrib/amcheck/verify_heapam.c @@ -75,7 +75,9 @@ typedef enum SkipPages */ typedef struct ToastedAttribute { - varatt_external_oid toast_pointer; + vartag_external tag; /* VARTAG_ONDISK_OID or VARTAG_ONDISK_OID8 */ + Oid8 va_valueid; /* value ID (works for both Oid and Oid8) */ + uint32 va_extinfo; /* external size and compression method */ BlockNumber blkno; /* block in main table */ OffsetNumber offnum; /* offset in main table */ AttrNumber attnum; /* attribute in main table */ @@ -1566,9 +1568,12 @@ check_toast_tuple(HeapTuple toasttup, HeapCheckContext *ctx, int32 max_chunk_size; Oid8 toast_valueid; - max_chunk_size = TOAST_OID_MAX_CHUNK_SIZE; + toast_valueid = ta->va_valueid; + + max_chunk_size = ta->tag == VARTAG_ONDISK_OID8 + ? TOAST_OID8_MAX_CHUNK_SIZE + : TOAST_OID_MAX_CHUNK_SIZE; last_chunk_seq = (extsize - 1) / max_chunk_size; - toast_valueid = ta->toast_pointer.va_valueid; /* Sanity-check the sequence number. */ chunk_seq = DatumGetInt32(fastgetattr(toasttup, 2, @@ -1672,8 +1677,10 @@ check_tuple_attribute(HeapCheckContext *ctx) char *tp; /* pointer to the tuple data */ uint16 infomask; Oid8 toast_pointer_valueid; + int32 va_rawsize; + uint32 va_extinfo; CompactAttribute *thisatt; - varatt_external_oid toast_pointer; + vartag_external va_tag_value; infomask = ctx->tuphdr->t_infomask; thisatt = TupleDescCompactAttr(RelationGetDescr(ctx->rel), ctx->attnum); @@ -1732,7 +1739,7 @@ check_tuple_attribute(HeapCheckContext *ctx) { uint8 va_tag = VARTAG_EXTERNAL(tp + ctx->offset); - if (va_tag != VARTAG_ONDISK_OID) + if (va_tag != VARTAG_ONDISK_OID && va_tag != VARTAG_ONDISK_OID8) { report_corruption(ctx, psprintf("toasted attribute has unexpected TOAST tag %u", @@ -1777,26 +1784,46 @@ check_tuple_attribute(HeapCheckContext *ctx) /* It is external, and we're looking at a page on disk */ /* - * Must copy attr into toast_pointer for alignment considerations + * Must copy attr into toast_pointer for alignment considerations. + * Branch on the tag to determine which pointer type to extract. */ - VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); - toast_pointer_valueid = toast_pointer.va_valueid; + va_tag_value = VARTAG_EXTERNAL(attr); + if (va_tag_value == VARTAG_ONDISK_OID8) + { + varatt_external_oid8 toast_pointer8; + + /* Must copy to access aligned fields */ + VARATT_EXTERNAL_GET_POINTER(toast_pointer8, attr); + toast_pointer_valueid = VARATT_EXTERNAL_OID8_GET_VALUEID(toast_pointer8); + va_rawsize = toast_pointer8.va_rawsize; + va_extinfo = toast_pointer8.va_extinfo; + } + else + { + varatt_external_oid toast_pointer; + + /* Must copy to access aligned fields */ + VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + toast_pointer_valueid = toast_pointer.va_valueid; + va_rawsize = toast_pointer.va_rawsize; + va_extinfo = toast_pointer.va_extinfo; + } /* Toasted attributes too large to be untoasted should never be stored */ - if (toast_pointer.va_rawsize > VARLENA_SIZE_LIMIT) + if (va_rawsize > VARLENA_SIZE_LIMIT) report_corruption(ctx, psprintf("toast value " OID8_FORMAT " rawsize %d exceeds limit %d", toast_pointer_valueid, - toast_pointer.va_rawsize, + va_rawsize, VARLENA_SIZE_LIMIT)); - if (VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer)) + if ((va_extinfo & VARLENA_EXTSIZE_MASK) < (Size) (va_rawsize - VARHDRSZ)) { ToastCompressionId cmid; bool valid = false; /* Compressed attributes should have a valid compression method */ - cmid = TOAST_COMPRESS_METHOD(&toast_pointer); + cmid = va_extinfo >> VARLENA_EXTSIZE_BITS; switch (cmid) { /* List of all valid compression method IDs */ @@ -1850,7 +1877,23 @@ check_tuple_attribute(HeapCheckContext *ctx) ta = palloc0_object(ToastedAttribute); - VARATT_EXTERNAL_GET_POINTER(ta->toast_pointer, attr); + ta->tag = va_tag_value; + if (va_tag_value == VARTAG_ONDISK_OID8) + { + varatt_external_oid8 tp; + + VARATT_EXTERNAL_GET_POINTER(tp, attr); + ta->va_valueid = VARATT_EXTERNAL_OID8_GET_VALUEID(tp); + ta->va_extinfo = tp.va_extinfo; + } + else + { + varatt_external_oid tp; + + VARATT_EXTERNAL_GET_POINTER(tp, attr); + ta->va_valueid = tp.va_valueid; + ta->va_extinfo = tp.va_extinfo; + } ta->blkno = ctx->blkno; ta->offnum = ctx->offnum; ta->attnum = ctx->attnum; @@ -1878,29 +1921,33 @@ check_toasted_attribute(HeapCheckContext *ctx, ToastedAttribute *ta) int32 last_chunk_seq; int32 max_chunk_size; Oid8 toast_valueid; - Oid toast_typid; - toast_typid = TupleDescAttr(ctx->toast_rel->rd_att, 0)->atttypid; - max_chunk_size = TOAST_OID_MAX_CHUNK_SIZE; - - extsize = VARATT_EXTERNAL_OID_GET_EXTSIZE(ta->toast_pointer); - last_chunk_seq = (extsize - 1) / max_chunk_size; + toast_valueid = ta->va_valueid; + extsize = ta->va_extinfo & VARLENA_EXTSIZE_MASK; /* * Setup a scan key to find chunks in toast table with matching va_valueid */ - if (toast_typid == OIDOID) - ScanKeyInit(&toastkey, - (AttrNumber) 1, - BTEqualStrategyNumber, F_OIDEQ, - ObjectIdGetDatum(ta->toast_pointer.va_valueid)); - else if (toast_typid == OID8OID) + if (ta->tag == VARTAG_ONDISK_OID8) + { + max_chunk_size = TOAST_OID8_MAX_CHUNK_SIZE; + ScanKeyInit(&toastkey, (AttrNumber) 1, BTEqualStrategyNumber, F_OID8EQ, - ObjectId8GetDatum(ta->toast_pointer.va_valueid)); + ObjectId8GetDatum(toast_valueid)); + } else - Assert(false); + { + max_chunk_size = TOAST_OID_MAX_CHUNK_SIZE; + + ScanKeyInit(&toastkey, + (AttrNumber) 1, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum((Oid) toast_valueid)); + } + + last_chunk_seq = (extsize - 1) / max_chunk_size; /* * Check if any chunks for this toasted object exist in the toast table, @@ -1920,8 +1967,6 @@ check_toasted_attribute(HeapCheckContext *ctx, ToastedAttribute *ta) } systable_endscan_ordered(toastscan); - toast_valueid = ta->toast_pointer.va_valueid; - if (!found_toasttup) report_toast_corruption(ctx, ta, psprintf("toast value " OID8_FORMAT " not found in toast table", -- 2.55.0