From 8e6f816cd51acffa681a4ac561af510a4e019cef Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Mon, 23 Jun 2025 12:43:14 +0900 Subject: [PATCH v2 03/13] Refactor external TOAST pointer code for better pluggability This commit introduces a new interface for external TOAST pointers, which is able to make a translation of the varlena pointers stored on disk to/from an new in-memory structure called toast_external. The types of varatt_external supported on disk need to be registered into a new subsystem in a new file, called toast_external.[c|h], then define a set of callbacks to allow the toasting and detoasting code to use it. The existing varatt_external is renamed to varatt_external_oid, to map with the fact that this structure is used to store external TOAST pointers based on OID values. A follow-up change will rely on this refactoring to introduce a new type of vartag_external with an associated varatt_external that is able to store 8-byte value IDs on disk. --- src/include/access/detoast.h | 12 +- src/include/access/heaptoast.h | 5 +- src/include/access/toast_external.h | 163 ++++++++++++++++++ src/include/access/toast_helper.h | 1 + src/include/varatt.h | 34 ++-- src/backend/access/common/Makefile | 1 + src/backend/access/common/detoast.c | 59 +++---- src/backend/access/common/meson.build | 1 + src/backend/access/common/toast_compression.c | 10 +- src/backend/access/common/toast_external.c | 131 ++++++++++++++ src/backend/access/common/toast_internals.c | 85 +++++---- src/backend/access/heap/heaptoast.c | 30 +++- src/backend/access/table/toast_helper.c | 12 +- .../replication/logical/reorderbuffer.c | 13 +- src/backend/utils/adt/varlena.c | 7 +- src/backend/utils/cache/relcache.c | 1 + doc/src/sgml/storage.sgml | 2 +- contrib/amcheck/verify_heapam.c | 37 ++-- src/tools/pgindent/typedefs.list | 2 + 19 files changed, 489 insertions(+), 117 deletions(-) create mode 100644 src/include/access/toast_external.h create mode 100644 src/backend/access/common/toast_external.c diff --git a/src/include/access/detoast.h b/src/include/access/detoast.h index e603a2276c38..4195f7b5bdfd 100644 --- a/src/include/access/detoast.h +++ b/src/include/access/detoast.h @@ -14,10 +14,11 @@ /* * Macro to fetch the possibly-unaligned contents of an EXTERNAL datum - * into a local "struct varatt_external" toast pointer. This should be - * just a memcpy, but some versions of gcc seem to produce broken code - * that assumes the datum contents are aligned. Introducing an explicit - * intermediate "varattrib_1b_e *" variable seems to fix it. + * into a local "struct varatt_external_*" toast pointer, as supported + * in toast_external.c and varatt.h. This should be just a memcpy, but + * some versions of gcc seem to produce broken code that assumes the datum + * contents are aligned. Introducing an explicit intermediate + * "varattrib_1b_e *" variable seems to fix it. */ #define VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr) \ do { \ @@ -27,9 +28,6 @@ do { \ memcpy(&(toast_pointer), VARDATA_EXTERNAL(attre), sizeof(toast_pointer)); \ } while (0) -/* Size of an EXTERNAL datum that contains a standard TOAST pointer */ -#define TOAST_POINTER_SIZE (VARHDRSZ_EXTERNAL + sizeof(varatt_external)) - /* Size of an EXTERNAL datum that contains an indirection pointer */ #define INDIRECT_POINTER_SIZE (VARHDRSZ_EXTERNAL + sizeof(varatt_indirect)) diff --git a/src/include/access/heaptoast.h b/src/include/access/heaptoast.h index 6e3558cbd6d2..673e96f5488c 100644 --- a/src/include/access/heaptoast.h +++ b/src/include/access/heaptoast.h @@ -81,13 +81,16 @@ #define EXTERN_TUPLE_MAX_SIZE MaximumBytesPerTuple(EXTERN_TUPLES_PER_PAGE) -#define TOAST_MAX_CHUNK_SIZE \ +#define TOAST_MAX_CHUNK_SIZE_OID \ (EXTERN_TUPLE_MAX_SIZE - \ MAXALIGN(SizeofHeapTupleHeader) - \ sizeof(Oid) - \ sizeof(int32) - \ VARHDRSZ) +/* Maximum size of chunk possible */ +#define TOAST_MAX_CHUNK_SIZE TOAST_MAX_CHUNK_SIZE_OID + /* ---------- * heap_toast_insert_or_update - * diff --git a/src/include/access/toast_external.h b/src/include/access/toast_external.h new file mode 100644 index 000000000000..1e3ba8062286 --- /dev/null +++ b/src/include/access/toast_external.h @@ -0,0 +1,163 @@ +/*------------------------------------------------------------------------- + * + * toast_external.h + * Support for on-disk external TOAST pointers + * + * + * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group + * Portions Copyright (c) 1995, Regents of the University of California + * + * src/include/access/toast_external.h + * + *------------------------------------------------------------------------- + */ + +#ifndef TOAST_EXTERNAL_H +#define TOAST_EXTERNAL_H + +#include "access/toast_compression.h" +#include "varatt.h" + +/* + * Intermediate in-memory structure used when creating on-disk + * varatt_external_* or when deserializing varlena contents. + */ +typedef struct toast_external_data +{ + /* Original data size (includes header) */ + int32 rawsize; + /* External saved size (without header) */ + uint32 extsize; + /* compression method */ + ToastCompressionId compression_method; + /* Relation OID of TOAST table containing the value */ + Oid toastrelid; + /* + * Unique ID of value within TOAST table. This could be an OID or an + * int8 value. This field is large enough to be able to store any of + * them. + */ + uint64 value; +} toast_external_data; + +/* + * Metadata for external TOAST pointer kinds, separated based on their + * vartag_external. + */ +typedef struct toast_external_info +{ + /* + * Maximum chunk of data authorized for this type of external TOAST + * pointer, when dividing an entry by chunks. Sized depending on + * the size of its varatt_external_* structure. + */ + int32 maximum_chunk_size; + + /* + * Size of an external TOAST pointer of this type, typically + * (VARHDRSZ_EXTERNAL + sizeof(varatt_external_struct)). + */ + int32 toast_pointer_size; + + /* + * Map an input varlena to a toast_external_data, for consumption + * in the backend code. "data" is an input/output result. + */ + void (*to_external_data) (struct varlena *attr, + toast_external_data *data); + + /* + * Create a varlena that will be used on-disk for the given TOAST + * type, based on the given input data. + * + * The result is the varlena created, for on-disk insertion. + */ + struct varlena *(*create_external_data) (toast_external_data data); + +} toast_external_info; + +/* Retrieve a toast_external_info from a vartag */ +extern const toast_external_info *toast_external_get_info(uint8 tag); + +/* Retrieve toast_pointer_size using a TOAST attribute type */ +extern int32 toast_external_info_get_pointer_size(uint8 tag); + +/* + * Testing whether an externally-stored value is compressed now requires + * comparing size stored in extsize (the actual length of the external data) + * to rawsize (the original uncompressed datum's size). The latter includes + * VARHDRSZ overhead, the former doesn't. We never use compression unless it + * actually saves space, so we expect either equality or less-than. + */ +#define TOAST_EXTERNAL_IS_COMPRESSED(data) \ + ((data).extsize < (data).rawsize - VARHDRSZ) + +/* Full data structure */ +static inline void +toast_external_info_get_data(struct varlena *attr, toast_external_data *data) +{ + uint8 tag = VARTAG_EXTERNAL(attr); + const toast_external_info *info = toast_external_get_info(tag); + + info->to_external_data(attr, data); +} + +/* + * Helper routines to recover specific fields in toast_external_data. Most + * code paths doing work with on-disk external TOAST pointers care about + * these. + */ + +/* Detoasted "raw" size */ +static inline Size +toast_external_info_get_rawsize(struct varlena *attr) +{ + uint8 tag = VARTAG_EXTERNAL(attr); + const toast_external_info *info = toast_external_get_info(tag); + toast_external_data data; + + info->to_external_data(attr, &data); + + return data.rawsize; +} + +/* External saved size */ +static inline Size +toast_external_info_get_extsize(struct varlena *attr) +{ + uint8 tag = VARTAG_EXTERNAL(attr); + const toast_external_info *info = toast_external_get_info(tag); + toast_external_data data; + + info->to_external_data(attr, &data); + + return data.extsize; +} + +/* Compression method ID */ +static inline ToastCompressionId +toast_external_info_get_compression_method(struct varlena *attr) +{ + uint8 tag = VARTAG_EXTERNAL(attr); + const toast_external_info *info = toast_external_get_info(tag); + toast_external_data data; + + info->to_external_data(attr, &data); + + return data.compression_method; +} + +/* Value ID */ +static inline Size +toast_external_info_get_value(struct varlena *attr) +{ + uint8 tag = VARTAG_EXTERNAL(attr); + const toast_external_info *info = toast_external_get_info(tag); + toast_external_data data; + + info->to_external_data(attr, &data); + + return data.value; +} + +#endif /* TOAST_EXTERNAL_H */ diff --git a/src/include/access/toast_helper.h b/src/include/access/toast_helper.h index e6ab8afffb67..729c593afebd 100644 --- a/src/include/access/toast_helper.h +++ b/src/include/access/toast_helper.h @@ -47,6 +47,7 @@ typedef struct * should be NULL in the case of an insert. */ Relation ttc_rel; /* the relation that contains the tuple */ + int32 ttc_toast_pointer_size; /* size of external TOAST pointer */ Datum *ttc_values; /* values from the tuple columns */ bool *ttc_isnull; /* null flags for the tuple columns */ Datum *ttc_oldvalues; /* values from previous tuple */ diff --git a/src/include/varatt.h b/src/include/varatt.h index 2e8564d49980..793030dae932 100644 --- a/src/include/varatt.h +++ b/src/include/varatt.h @@ -16,11 +16,14 @@ #define VARATT_H /* - * struct varatt_external is a traditional "TOAST pointer", that is, the + * struct varatt_external_oid is a traditional "TOAST pointer", that is, the * information needed to fetch a Datum stored out-of-line in a TOAST table. * The data is compressed if and only if the external size stored in * va_extinfo is less than va_rawsize - VARHDRSZ. * + * The value ID used is an OID, used for TOAST relations with OID as + * attribute for chunk_id. + * * This struct must not contain any padding, because we sometimes compare * these pointers using memcmp. * @@ -29,14 +32,15 @@ * you can look at these fields! (The reason we use memcmp is to avoid * having to do that just to detect equality of two TOAST pointers...) */ -typedef struct varatt_external +typedef struct varatt_external_oid { int32 va_rawsize; /* Original data size (includes header) */ uint32 va_extinfo; /* External saved size (without header) and * compression method */ Oid va_valueid; /* Unique ID of value within TOAST table */ Oid va_toastrelid; /* RelID of TOAST table containing it */ -} varatt_external; +} varatt_external_oid; + /* * These macros define the "saved size" portion of va_extinfo. Its remaining @@ -78,15 +82,16 @@ typedef struct varatt_expanded /* * Type tag for the various sorts of "TOAST pointer" datums. The peculiar - * value for VARTAG_ONDISK comes from a requirement for on-disk compatibility - * with a previous notion that the tag field was the pointer datum's length. + * value for VARTAG_ONDISK_OID comes from a requirement for on-disk + * compatibility with a previous notion that the tag field was the pointer + * datum's length. */ typedef enum vartag_external { VARTAG_INDIRECT = 1, VARTAG_EXPANDED_RO = 2, VARTAG_EXPANDED_RW = 3, - VARTAG_ONDISK = 18 + VARTAG_ONDISK_OID = 18 } vartag_external; /* this test relies on the specific tag values above */ @@ -96,7 +101,7 @@ typedef enum vartag_external #define VARTAG_SIZE(tag) \ ((tag) == VARTAG_INDIRECT ? sizeof(varatt_indirect) : \ VARTAG_IS_EXPANDED(tag) ? sizeof(varatt_expanded) : \ - (tag) == VARTAG_ONDISK ? sizeof(varatt_external) : \ + (tag) == VARTAG_ONDISK_OID ? sizeof(varatt_external_oid) : \ (AssertMacro(false), 0)) /* @@ -287,8 +292,10 @@ typedef struct #define VARATT_IS_COMPRESSED(PTR) VARATT_IS_4B_C(PTR) #define VARATT_IS_EXTERNAL(PTR) VARATT_IS_1B_E(PTR) +#define VARATT_IS_EXTERNAL_ONDISK_OID(PTR) \ + (VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_ONDISK_OID) #define VARATT_IS_EXTERNAL_ONDISK(PTR) \ - (VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_ONDISK) + (VARATT_IS_EXTERNAL_ONDISK_OID(PTR)) #define VARATT_IS_EXTERNAL_INDIRECT(PTR) \ (VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_INDIRECT) #define VARATT_IS_EXTERNAL_EXPANDED_RO(PTR) \ @@ -330,7 +337,10 @@ typedef struct #define VARDATA_COMPRESSED_GET_COMPRESS_METHOD(PTR) \ (((varattrib_4b *) (PTR))->va_compressed.va_tcinfo >> VARLENA_EXTSIZE_BITS) -/* Same for external Datums; but note argument is a struct varatt_external */ +/* + * Same for external Datums; but note argument is a struct + * varatt_external_oid. + */ #define VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer) \ ((toast_pointer).va_extinfo & VARLENA_EXTSIZE_MASK) #define VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer) \ @@ -351,8 +361,10 @@ typedef struct * VARHDRSZ overhead, the former doesn't. We never use compression unless it * actually saves space, so we expect either equality or less-than. */ + #define VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer) \ - (VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer) < \ - (toast_pointer).va_rawsize - VARHDRSZ) + (VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer) < \ + (toast_pointer).va_rawsize - VARHDRSZ) + #endif diff --git a/src/backend/access/common/Makefile b/src/backend/access/common/Makefile index e78de312659e..1ef86a245886 100644 --- a/src/backend/access/common/Makefile +++ b/src/backend/access/common/Makefile @@ -27,6 +27,7 @@ OBJS = \ syncscan.o \ tidstore.o \ toast_compression.o \ + toast_external.o \ toast_internals.o \ tupconvert.o \ tupdesc.o diff --git a/src/backend/access/common/detoast.c b/src/backend/access/common/detoast.c index 626517877422..684e1b0b7d3b 100644 --- a/src/backend/access/common/detoast.c +++ b/src/backend/access/common/detoast.c @@ -16,6 +16,7 @@ #include "access/detoast.h" #include "access/table.h" #include "access/tableam.h" +#include "access/toast_external.h" #include "access/toast_internals.h" #include "common/int.h" #include "common/pg_lzcompress.h" @@ -225,12 +226,12 @@ detoast_attr_slice(struct varlena *attr, if (VARATT_IS_EXTERNAL_ONDISK(attr)) { - struct varatt_external toast_pointer; + struct toast_external_data toast_pointer; - VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + toast_external_info_get_data(attr, &toast_pointer); /* fast path for non-compressed external datums */ - if (!VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer)) + if (!TOAST_EXTERNAL_IS_COMPRESSED(toast_pointer)) return toast_fetch_datum_slice(attr, sliceoffset, slicelength); /* @@ -240,7 +241,7 @@ detoast_attr_slice(struct varlena *attr, */ if (slicelimit >= 0) { - int32 max_size = VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer); + int32 max_size = toast_pointer.extsize; /* * Determine maximum amount of compressed data needed for a prefix @@ -251,8 +252,7 @@ detoast_attr_slice(struct varlena *attr, * determine how much compressed data we need to be sure of being * able to decompress the required slice. */ - if (VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer) == - TOAST_PGLZ_COMPRESSION_ID) + if (toast_pointer.compression_method == TOAST_PGLZ_COMPRESSION_ID) max_size = pglz_maximum_compressed_size(slicelimit, max_size); /* @@ -344,20 +344,21 @@ toast_fetch_datum(struct varlena *attr) { Relation toastrel; struct varlena *result; - struct varatt_external toast_pointer; + struct toast_external_data toast_pointer; int32 attrsize; + uint64 valueid; if (!VARATT_IS_EXTERNAL_ONDISK(attr)) elog(ERROR, "toast_fetch_datum shouldn't be called for non-ondisk datums"); /* Must copy to access aligned fields */ - VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + toast_external_info_get_data(attr, &toast_pointer); - attrsize = VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer); + attrsize = toast_pointer.extsize; result = (struct varlena *) palloc(attrsize + VARHDRSZ); - if (VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer)) + if (TOAST_EXTERNAL_IS_COMPRESSED(toast_pointer)) SET_VARSIZE_COMPRESSED(result, attrsize + VARHDRSZ); else SET_VARSIZE(result, attrsize + VARHDRSZ); @@ -365,14 +366,15 @@ toast_fetch_datum(struct varlena *attr) if (attrsize == 0) return result; /* Probably shouldn't happen, but just in * case. */ + valueid = toast_pointer.value; /* * Open the toast relation and its indexes */ - toastrel = table_open(toast_pointer.va_toastrelid, AccessShareLock); + toastrel = table_open(toast_pointer.toastrelid, AccessShareLock); /* Fetch all chunks */ - table_relation_fetch_toast_slice(toastrel, toast_pointer.va_valueid, + table_relation_fetch_toast_slice(toastrel, valueid, attrsize, 0, attrsize, result); /* Close toast table */ @@ -398,23 +400,26 @@ toast_fetch_datum_slice(struct varlena *attr, int32 sliceoffset, { Relation toastrel; struct varlena *result; - struct varatt_external toast_pointer; + struct toast_external_data toast_pointer; int32 attrsize; + uint64 valueid; if (!VARATT_IS_EXTERNAL_ONDISK(attr)) elog(ERROR, "toast_fetch_datum_slice shouldn't be called for non-ondisk datums"); /* Must copy to access aligned fields */ - VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + toast_external_info_get_data(attr, &toast_pointer); + + valueid = toast_pointer.value; /* * It's nonsense to fetch slices of a compressed datum unless when it's a * prefix -- this isn't lo_* we can't return a compressed datum which is * meaningful to toast later. */ - Assert(!VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer) || 0 == sliceoffset); + Assert(!TOAST_EXTERNAL_IS_COMPRESSED(toast_pointer) || 0 == sliceoffset); - attrsize = VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer); + attrsize = toast_pointer.extsize; if (sliceoffset >= attrsize) { @@ -427,7 +432,7 @@ toast_fetch_datum_slice(struct varlena *attr, int32 sliceoffset, * space required by va_tcinfo, which is stored at the beginning as an * int32 value. */ - if (VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer) && slicelength > 0) + if (TOAST_EXTERNAL_IS_COMPRESSED(toast_pointer) && slicelength > 0) slicelength = slicelength + sizeof(int32); /* @@ -440,7 +445,7 @@ toast_fetch_datum_slice(struct varlena *attr, int32 sliceoffset, result = (struct varlena *) palloc(slicelength + VARHDRSZ); - if (VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer)) + if (TOAST_EXTERNAL_IS_COMPRESSED(toast_pointer)) SET_VARSIZE_COMPRESSED(result, slicelength + VARHDRSZ); else SET_VARSIZE(result, slicelength + VARHDRSZ); @@ -449,10 +454,11 @@ toast_fetch_datum_slice(struct varlena *attr, int32 sliceoffset, return result; /* Can save a lot of work at this point! */ /* Open the toast relation */ - toastrel = table_open(toast_pointer.va_toastrelid, AccessShareLock); + toastrel = table_open(toast_pointer.toastrelid, AccessShareLock); /* Fetch all chunks */ - table_relation_fetch_toast_slice(toastrel, toast_pointer.va_valueid, + table_relation_fetch_toast_slice(toastrel, + valueid, attrsize, sliceoffset, slicelength, result); @@ -549,11 +555,7 @@ toast_raw_datum_size(Datum value) if (VARATT_IS_EXTERNAL_ONDISK(attr)) { - /* va_rawsize is the size of the original datum -- including header */ - struct varatt_external toast_pointer; - - VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); - result = toast_pointer.va_rawsize; + result = toast_external_info_get_rawsize(attr); } else if (VARATT_IS_EXTERNAL_INDIRECT(attr)) { @@ -609,11 +611,10 @@ toast_datum_size(Datum value) * Attribute is stored externally - return the extsize whether * compressed or not. We do not count the size of the toast pointer * ... should we? + * + * XXX: this comment should be documented elsewhere. */ - struct varatt_external toast_pointer; - - VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); - result = VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer); + result = toast_external_info_get_extsize(attr); } else if (VARATT_IS_EXTERNAL_INDIRECT(attr)) { diff --git a/src/backend/access/common/meson.build b/src/backend/access/common/meson.build index e3cdbe7a22e1..c20f2e88921e 100644 --- a/src/backend/access/common/meson.build +++ b/src/backend/access/common/meson.build @@ -15,6 +15,7 @@ backend_sources += files( 'syncscan.c', 'tidstore.c', 'toast_compression.c', + 'toast_external.c', 'toast_internals.c', 'tupconvert.c', 'tupdesc.c', diff --git a/src/backend/access/common/toast_compression.c b/src/backend/access/common/toast_compression.c index 21f2f4af97e3..e1c76dea4905 100644 --- a/src/backend/access/common/toast_compression.c +++ b/src/backend/access/common/toast_compression.c @@ -19,6 +19,7 @@ #include "access/detoast.h" #include "access/toast_compression.h" +#include "access/toast_external.h" #include "common/pg_lzcompress.h" #include "varatt.h" @@ -261,14 +262,7 @@ toast_get_compression_id(struct varlena *attr) * toast compression header. */ if (VARATT_IS_EXTERNAL_ONDISK(attr)) - { - struct varatt_external toast_pointer; - - VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); - - if (VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer)) - cmid = VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer); - } + cmid = toast_external_info_get_compression_method(attr); else if (VARATT_IS_COMPRESSED(attr)) cmid = VARDATA_COMPRESSED_GET_COMPRESS_METHOD(attr); diff --git a/src/backend/access/common/toast_external.c b/src/backend/access/common/toast_external.c new file mode 100644 index 000000000000..a851a8e4184b --- /dev/null +++ b/src/backend/access/common/toast_external.c @@ -0,0 +1,131 @@ +/*------------------------------------------------------------------------- + * + * toast_external.c + * Functions for the support of external on-disk TOAST pointers. + * + * Copyright (c) 2025, PostgreSQL Global Development Group + * + * + * IDENTIFICATION + * src/backend/access/common/toast_external.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/detoast.h" +#include "access/heaptoast.h" +#include "access/toast_external.h" + +/* Callbacks for VARTAG_ONDISK_OID */ +static void ondisk_oid_to_external_data(struct varlena *attr, + toast_external_data *data); +static struct varlena *ondisk_oid_create_external_data(toast_external_data data); + + +/* + * Size of an EXTERNAL datum that contains a standard TOAST pointer (OID + * value). + */ +#define TOAST_POINTER_OID_SIZE (VARHDRSZ_EXTERNAL + sizeof(varatt_external_oid)) + +/* + * For now there are only two types, all defined in this file. For now this + * is the maximum value of vartag_external, which is a historical choice. + */ +#define TOAST_EXTERNAL_INFO_SIZE (VARTAG_ONDISK_OID + 1) + +/* + * The different kinds of on-disk external TOAST pointers. divided by + * vartag_external. + * + * See comments for struct toast_external_info about the details of the + * individual fields. + */ +static const toast_external_info toast_external_infos[TOAST_EXTERNAL_INFO_SIZE] = { + [VARTAG_ONDISK_OID] = { + .toast_pointer_size = TOAST_POINTER_OID_SIZE, + .maximum_chunk_size = TOAST_MAX_CHUNK_SIZE_OID, + .to_external_data = ondisk_oid_to_external_data, + .create_external_data = ondisk_oid_create_external_data, + }, +}; + + +/* Get toast_external_info of the defined vartag_external */ +const toast_external_info * +toast_external_get_info(uint8 tag) +{ + return &toast_external_infos[tag]; +} + +/* + * Get external TOAST pointer size based on the attribute type of a TOAST + * value. + */ +int32 +toast_external_info_get_pointer_size(uint8 tag) +{ + return toast_external_infos[tag].toast_pointer_size; +} + +/* + * Helper routines able to translate the various varatt_external_* from/to + * the in-memory representation toast_external_data used in the backend. + */ + +/* Callbacks for VARTAG_ONDISK_OID */ +static void +ondisk_oid_to_external_data(struct varlena *attr, toast_external_data *data) +{ + varatt_external_oid external; + + VARATT_EXTERNAL_GET_POINTER(external, attr); + data->rawsize = external.va_rawsize; + + /* + * External size and compression methods are stored in the same field, + * extract. + */ + if (VARATT_EXTERNAL_IS_COMPRESSED(external)) + { + data->extsize = VARATT_EXTERNAL_GET_EXTSIZE(external); + data->compression_method = VARATT_EXTERNAL_GET_COMPRESS_METHOD(external); + } + else + { + data->extsize = external.va_extinfo; + data->compression_method = TOAST_INVALID_COMPRESSION_ID; + } + + data->value = (uint64) external.va_valueid; + data->toastrelid = external.va_toastrelid; +} + +static struct varlena * +ondisk_oid_create_external_data(toast_external_data data) +{ + struct varlena *result = NULL; + varatt_external_oid external; + + external.va_rawsize = data.rawsize; + + if (data.compression_method != TOAST_INVALID_COMPRESSION_ID) + { + /* Set size and compression method, in a single field. */ + VARATT_EXTERNAL_SET_SIZE_AND_COMPRESS_METHOD(external, + data.extsize, + data.compression_method); + } + else + external.va_extinfo = data.extsize; + + external.va_toastrelid = data.toastrelid; + external.va_valueid = (Oid) data.value; + + result = (struct varlena *) palloc(TOAST_POINTER_OID_SIZE); + SET_VARTAG_EXTERNAL(result, VARTAG_ONDISK_OID); + memcpy(VARDATA_EXTERNAL(result), &external, sizeof(external)); + + return result; +} diff --git a/src/backend/access/common/toast_internals.c b/src/backend/access/common/toast_internals.c index 4a1342da6e1b..5e6e19fbc363 100644 --- a/src/backend/access/common/toast_internals.c +++ b/src/backend/access/common/toast_internals.c @@ -18,6 +18,7 @@ #include "access/heapam.h" #include "access/heaptoast.h" #include "access/table.h" +#include "access/toast_external.h" #include "access/toast_internals.h" #include "access/xact.h" #include "catalog/catalog.h" @@ -127,7 +128,7 @@ toast_save_datum(Relation rel, Datum value, bool t_isnull[3]; CommandId mycid = GetCurrentCommandId(true); struct varlena *result; - struct varatt_external toast_pointer; + struct toast_external_data toast_pointer; union { struct varlena hdr; @@ -143,6 +144,8 @@ toast_save_datum(Relation rel, Datum value, Pointer dval = DatumGetPointer(value); int num_indexes; int validIndex; + const toast_external_info *info; + uint8 tag = VARTAG_INDIRECT; /* init value does not matter */ Assert(!VARATT_IS_EXTERNAL(value)); @@ -154,6 +157,15 @@ toast_save_datum(Relation rel, Datum value, toastrel = table_open(rel->rd_rel->reltoastrelid, RowExclusiveLock); toasttupDesc = toastrel->rd_att; + /* + * Grab the information for toast_external_data. + * + * Note: if we support multiple external vartags for a single value + * type, we would need to be smarter in the vartag selection. + */ + tag = VARTAG_ONDISK_OID; + info = toast_external_get_info(tag); + /* Open all the toast indexes and look for the valid one */ validIndex = toast_open_indexes(toastrel, RowExclusiveLock, @@ -174,28 +186,41 @@ toast_save_datum(Relation rel, Datum value, { data_p = VARDATA_SHORT(dval); data_todo = VARSIZE_SHORT(dval) - VARHDRSZ_SHORT; - toast_pointer.va_rawsize = data_todo + VARHDRSZ; /* as if not short */ - toast_pointer.va_extinfo = data_todo; + toast_pointer.rawsize = data_todo + VARHDRSZ; /* as if not short */ + toast_pointer.extsize = data_todo; + + /* + * Note: we set compression_method to be able to build a correct + * on-disk TOAST pointer. + */ + toast_pointer.compression_method = TOAST_INVALID_COMPRESSION_ID; } else if (VARATT_IS_COMPRESSED(dval)) { data_p = VARDATA(dval); data_todo = VARSIZE(dval) - VARHDRSZ; /* rawsize in a compressed datum is just the size of the payload */ - toast_pointer.va_rawsize = VARDATA_COMPRESSED_GET_EXTSIZE(dval) + VARHDRSZ; + toast_pointer.rawsize = VARDATA_COMPRESSED_GET_EXTSIZE(dval) + VARHDRSZ; /* set external size and compression method */ - VARATT_EXTERNAL_SET_SIZE_AND_COMPRESS_METHOD(toast_pointer, data_todo, - VARDATA_COMPRESSED_GET_COMPRESS_METHOD(dval)); + toast_pointer.extsize = data_todo; + toast_pointer.compression_method = VARDATA_COMPRESSED_GET_COMPRESS_METHOD(dval); + /* Assert that the numbers look like it's compressed */ - Assert(VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer)); + Assert(TOAST_EXTERNAL_IS_COMPRESSED(toast_pointer)); } else { data_p = VARDATA(dval); data_todo = VARSIZE(dval) - VARHDRSZ; - toast_pointer.va_rawsize = VARSIZE(dval); - toast_pointer.va_extinfo = data_todo; + toast_pointer.rawsize = VARSIZE(dval); + toast_pointer.extsize = data_todo; + + /* + * Note: we set compression_method to be able to build a correct + * on-disk TOAST pointer. + */ + toast_pointer.compression_method = TOAST_INVALID_COMPRESSION_ID; } /* @@ -207,9 +232,9 @@ toast_save_datum(Relation rel, Datum value, * if we have to substitute such an OID. */ if (OidIsValid(rel->rd_toastoid)) - toast_pointer.va_toastrelid = rel->rd_toastoid; + toast_pointer.toastrelid = rel->rd_toastoid; else - toast_pointer.va_toastrelid = RelationGetRelid(toastrel); + toast_pointer.toastrelid = RelationGetRelid(toastrel); /* * Choose an OID to use as the value ID for this toast value. @@ -226,7 +251,7 @@ toast_save_datum(Relation rel, Datum value, if (!OidIsValid(rel->rd_toastoid)) { /* normal case: just choose an unused OID */ - toast_pointer.va_valueid = + toast_pointer.value = GetNewOidWithIndex(toastrel, RelationGetRelid(toastidxs[validIndex]), (AttrNumber) 1); @@ -234,18 +259,18 @@ toast_save_datum(Relation rel, Datum value, else { /* rewrite case: check to see if value was in old toast table */ - toast_pointer.va_valueid = InvalidOid; + toast_pointer.value = InvalidOid; if (oldexternal != NULL) { - struct varatt_external old_toast_pointer; + struct toast_external_data old_toast_pointer; Assert(VARATT_IS_EXTERNAL_ONDISK(oldexternal)); - /* Must copy to access aligned fields */ - VARATT_EXTERNAL_GET_POINTER(old_toast_pointer, oldexternal); - if (old_toast_pointer.va_toastrelid == rel->rd_toastoid) + toast_external_info_get_data(oldexternal, &old_toast_pointer); + + if (old_toast_pointer.toastrelid == rel->rd_toastoid) { /* This value came from the old toast table; reuse its OID */ - toast_pointer.va_valueid = old_toast_pointer.va_valueid; + toast_pointer.value = old_toast_pointer.value; /* * There is a corner case here: the table rewrite might have @@ -265,14 +290,14 @@ toast_save_datum(Relation rel, Datum value, * be reclaimed by VACUUM. */ if (toastrel_valueid_exists(toastrel, - toast_pointer.va_valueid)) + toast_pointer.value)) { /* Match, so short-circuit the data storage loop below */ data_todo = 0; } } } - if (toast_pointer.va_valueid == InvalidOid) + if (toast_pointer.value == InvalidOid) { /* * new value; must choose an OID that doesn't conflict in either @@ -280,19 +305,19 @@ toast_save_datum(Relation rel, Datum value, */ do { - toast_pointer.va_valueid = + toast_pointer.value = GetNewOidWithIndex(toastrel, RelationGetRelid(toastidxs[validIndex]), (AttrNumber) 1); } while (toastid_valueid_exists(rel->rd_toastoid, - toast_pointer.va_valueid)); + toast_pointer.value)); } } /* * Initialize constant parts of the tuple data */ - t_values[0] = ObjectIdGetDatum(toast_pointer.va_valueid); + t_values[0] = ObjectIdGetDatum(toast_pointer.value); t_values[2] = PointerGetDatum(&chunk_data); t_isnull[0] = false; t_isnull[1] = false; @@ -310,7 +335,7 @@ toast_save_datum(Relation rel, Datum value, /* * Calculate the size of this chunk */ - chunk_size = Min(TOAST_MAX_CHUNK_SIZE, data_todo); + chunk_size = Min(info->maximum_chunk_size, data_todo); /* * Build a tuple and store it @@ -368,9 +393,7 @@ toast_save_datum(Relation rel, Datum value, /* * Create the TOAST pointer value that we'll return */ - result = (struct varlena *) palloc(TOAST_POINTER_SIZE); - SET_VARTAG_EXTERNAL(result, VARTAG_ONDISK); - memcpy(VARDATA_EXTERNAL(result), &toast_pointer, sizeof(toast_pointer)); + result = info->create_external_data(toast_pointer); return PointerGetDatum(result); } @@ -385,7 +408,7 @@ void toast_delete_datum(Relation rel, Datum value, bool is_speculative) { struct varlena *attr = (struct varlena *) DatumGetPointer(value); - struct varatt_external toast_pointer; + struct toast_external_data toast_pointer; Relation toastrel; Relation *toastidxs; ScanKeyData toastkey; @@ -398,12 +421,12 @@ toast_delete_datum(Relation rel, Datum value, bool is_speculative) return; /* Must copy to access aligned fields */ - VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + toast_external_info_get_data(attr, &toast_pointer); /* * Open the toast relation and its indexes */ - toastrel = table_open(toast_pointer.va_toastrelid, RowExclusiveLock); + toastrel = table_open(toast_pointer.toastrelid, RowExclusiveLock); /* Fetch valid relation used for process */ validIndex = toast_open_indexes(toastrel, @@ -417,7 +440,7 @@ toast_delete_datum(Relation rel, Datum value, bool is_speculative) ScanKeyInit(&toastkey, (AttrNumber) 1, BTEqualStrategyNumber, F_OIDEQ, - ObjectIdGetDatum(toast_pointer.va_valueid)); + ObjectIdGetDatum(toast_pointer.value)); /* * Find all the chunks. (We don't actually care whether we see them in diff --git a/src/backend/access/heap/heaptoast.c b/src/backend/access/heap/heaptoast.c index ae8d502ddcd3..b3a63c10aef3 100644 --- a/src/backend/access/heap/heaptoast.c +++ b/src/backend/access/heap/heaptoast.c @@ -28,6 +28,7 @@ #include "access/genam.h" #include "access/heapam.h" #include "access/heaptoast.h" +#include "access/toast_external.h" #include "access/toast_helper.h" #include "access/toast_internals.h" #include "utils/fmgroids.h" @@ -140,6 +141,17 @@ heap_toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, * Prepare for toasting * ---------- */ + + /* + * Retrieve the toast pointer size based on the type of external TOAST + * pointer assumed to be used. + * + * Only one format of external TOAST pointer is supported currently, + * making this code simple, based on a single vartag. + */ + ttc.ttc_toast_pointer_size = + toast_external_info_get_pointer_size(VARTAG_ONDISK_OID); + ttc.ttc_rel = rel; ttc.ttc_values = toast_values; ttc.ttc_isnull = toast_isnull; @@ -640,6 +652,8 @@ heap_fetch_toast_slice(Relation toastrel, uint64 valueid, int32 attrsize, int num_indexes; int validIndex; int32 max_chunk_size; + const toast_external_info *info; + uint8 tag = VARTAG_INDIRECT; /* init value does not matter */ /* Look for the valid index of toast relation */ validIndex = toast_open_indexes(toastrel, @@ -647,7 +661,21 @@ heap_fetch_toast_slice(Relation toastrel, uint64 valueid, int32 attrsize, &toastidxs, &num_indexes); - max_chunk_size = TOAST_MAX_CHUNK_SIZE; + /* + * Grab the information for toast_external_data. + * + * Note: there is no access to the vartag of the original varlena from + * which we are trying to retrieve the chunks from the TOAST relation, + * so guess the external TOAST pointer information to use depending + * on the attribute of the TOAST value. If we begin to support multiple + * external TOAST pointers for a single attribute type, we would need + * to pass down this information from the upper callers. This is + * currently on required for the maximum chunk_size. + */ + tag = VARTAG_ONDISK_OID; + info = toast_external_get_info(tag); + + max_chunk_size = info->maximum_chunk_size; totalchunks = ((attrsize - 1) / max_chunk_size) + 1; startchunk = sliceoffset / max_chunk_size; diff --git a/src/backend/access/table/toast_helper.c b/src/backend/access/table/toast_helper.c index b60fab0a4d29..a2b44e093d79 100644 --- a/src/backend/access/table/toast_helper.c +++ b/src/backend/access/table/toast_helper.c @@ -171,8 +171,10 @@ toast_tuple_init(ToastTupleContext *ttc) * The column must have attstorage EXTERNAL or EXTENDED if check_main is * false, and must have attstorage MAIN if check_main is true. * - * The column must have a minimum size of MAXALIGN(TOAST_POINTER_SIZE); - * if not, no benefit is to be expected by compressing it. + * The column must have a minimum size of MAXALIGN(tcc_toast_pointer_size); + * if not, no benefit is to be expected by compressing it. The TOAST + * pointer size is given by the caller, depending on the type of TOAST + * table we are dealing with. * * The return value is the index of the biggest suitable column, or * -1 if there is none. @@ -184,10 +186,14 @@ toast_tuple_find_biggest_attribute(ToastTupleContext *ttc, TupleDesc tupleDesc = ttc->ttc_rel->rd_att; int numAttrs = tupleDesc->natts; int biggest_attno = -1; - int32 biggest_size = MAXALIGN(TOAST_POINTER_SIZE); + int32 biggest_size = 0; int32 skip_colflags = TOASTCOL_IGNORE; int i; + /* Define the lower-bound */ + biggest_size = MAXALIGN(ttc->ttc_toast_pointer_size); + Assert(biggest_size != 0); + if (for_compression) skip_colflags |= TOASTCOL_INCOMPRESSIBLE; diff --git a/src/backend/replication/logical/reorderbuffer.c b/src/backend/replication/logical/reorderbuffer.c index c871708b5932..77939c7f849c 100644 --- a/src/backend/replication/logical/reorderbuffer.c +++ b/src/backend/replication/logical/reorderbuffer.c @@ -92,6 +92,7 @@ #include "access/detoast.h" #include "access/heapam.h" #include "access/rewriteheap.h" +#include "access/toast_external.h" #include "access/transam.h" #include "access/xact.h" #include "access/xlog_internal.h" @@ -5102,7 +5103,7 @@ ReorderBufferToastReplace(ReorderBuffer *rb, ReorderBufferTXN *txn, struct varlena *varlena; /* va_rawsize is the size of the original datum -- including header */ - struct varatt_external toast_pointer; + struct toast_external_data toast_pointer; struct varatt_indirect redirect_pointer; struct varlena *new_datum = NULL; struct varlena *reconstructed; @@ -5132,8 +5133,8 @@ ReorderBufferToastReplace(ReorderBuffer *rb, ReorderBufferTXN *txn, if (!VARATT_IS_EXTERNAL(varlena)) continue; - VARATT_EXTERNAL_GET_POINTER(toast_pointer, varlena); - toast_valueid = toast_pointer.va_valueid; + toast_external_info_get_data(varlena, &toast_pointer); + toast_valueid = toast_pointer.value; /* * Check whether the toast tuple changed, replace if so. @@ -5151,7 +5152,7 @@ ReorderBufferToastReplace(ReorderBuffer *rb, ReorderBufferTXN *txn, free[natt] = true; - reconstructed = palloc0(toast_pointer.va_rawsize); + reconstructed = palloc0(toast_pointer.rawsize); ent->reconstructed = reconstructed; @@ -5176,10 +5177,10 @@ ReorderBufferToastReplace(ReorderBuffer *rb, ReorderBufferTXN *txn, VARSIZE(chunk) - VARHDRSZ); data_done += VARSIZE(chunk) - VARHDRSZ; } - Assert(data_done == VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer)); + Assert(data_done == toast_pointer.extsize); /* make sure its marked as compressed or not */ - if (VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer)) + if (TOAST_EXTERNAL_IS_COMPRESSED(toast_pointer)) SET_VARSIZE_COMPRESSED(reconstructed, data_done + VARHDRSZ); else SET_VARSIZE(reconstructed, data_done + VARHDRSZ); diff --git a/src/backend/utils/adt/varlena.c b/src/backend/utils/adt/varlena.c index ffae8c23abfa..d76386407a08 100644 --- a/src/backend/utils/adt/varlena.c +++ b/src/backend/utils/adt/varlena.c @@ -19,6 +19,7 @@ #include "access/detoast.h" #include "access/toast_compression.h" +#include "access/toast_external.h" #include "catalog/pg_collation.h" #include "catalog/pg_type.h" #include "common/hashfn.h" @@ -4219,7 +4220,7 @@ pg_column_toast_chunk_id(PG_FUNCTION_ARGS) { int typlen; struct varlena *attr; - struct varatt_external toast_pointer; + uint64 toast_valueid; /* On first call, get the input type's typlen, and save at *fn_extra */ if (fcinfo->flinfo->fn_extra == NULL) @@ -4246,9 +4247,9 @@ pg_column_toast_chunk_id(PG_FUNCTION_ARGS) if (!VARATT_IS_EXTERNAL_ONDISK(attr)) PG_RETURN_NULL(); - VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + toast_valueid = toast_external_info_get_value(attr); - PG_RETURN_OID(toast_pointer.va_valueid); + PG_RETURN_OID(toast_valueid); } /* diff --git a/src/backend/utils/cache/relcache.c b/src/backend/utils/cache/relcache.c index 559ba9cdb2cd..8761762a6f3a 100644 --- a/src/backend/utils/cache/relcache.c +++ b/src/backend/utils/cache/relcache.c @@ -37,6 +37,7 @@ #include "access/sysattr.h" #include "access/table.h" #include "access/tableam.h" +#include "access/toast_external.h" #include "access/tupdesc_details.h" #include "access/xact.h" #include "catalog/binary_upgrade.h" diff --git a/doc/src/sgml/storage.sgml b/doc/src/sgml/storage.sgml index 61250799ec07..f3c6cd8860b5 100644 --- a/doc/src/sgml/storage.sgml +++ b/doc/src/sgml/storage.sgml @@ -415,7 +415,7 @@ described in more detail below. Out-of-line values are divided (after compression if used) into chunks of at -most TOAST_MAX_CHUNK_SIZE bytes (by default this value is chosen +most TOAST_MAX_CHUNK_SIZE_OID bytes (by default this value is chosen so that four chunk rows will fit on a page, making it about 2000 bytes). Each chunk is stored as a separate row in the TOAST table belonging to the owning table. Every diff --git a/contrib/amcheck/verify_heapam.c b/contrib/amcheck/verify_heapam.c index 3b2bdced4cdc..11c4507ae6e2 100644 --- a/contrib/amcheck/verify_heapam.c +++ b/contrib/amcheck/verify_heapam.c @@ -16,6 +16,7 @@ #include "access/multixact.h" #include "access/relation.h" #include "access/table.h" +#include "access/toast_external.h" #include "access/toast_internals.h" #include "access/visibilitymap.h" #include "access/xact.h" @@ -73,7 +74,8 @@ typedef enum SkipPages */ typedef struct ToastedAttribute { - struct varatt_external toast_pointer; + struct toast_external_data toast_pointer; + const toast_external_info *info; BlockNumber blkno; /* block in main table */ OffsetNumber offnum; /* offset in main table */ AttrNumber attnum; /* attribute in main table */ @@ -1564,9 +1566,9 @@ check_toast_tuple(HeapTuple toasttup, HeapCheckContext *ctx, uint64 toast_valueid; int32 max_chunk_size; - toast_valueid = ta->toast_pointer.va_valueid; + toast_valueid = ta->toast_pointer.value; - max_chunk_size = TOAST_MAX_CHUNK_SIZE; + max_chunk_size = ta->info->maximum_chunk_size; last_chunk_seq = (extsize - 1) / max_chunk_size; /* Sanity-check the sequence number. */ @@ -1672,7 +1674,7 @@ check_tuple_attribute(HeapCheckContext *ctx) uint16 infomask; uint64 toast_pointer_valueid; CompactAttribute *thisatt; - struct varatt_external toast_pointer; + struct toast_external_data toast_pointer; infomask = ctx->tuphdr->t_infomask; thisatt = TupleDescCompactAttr(RelationGetDescr(ctx->rel), ctx->attnum); @@ -1731,7 +1733,7 @@ check_tuple_attribute(HeapCheckContext *ctx) { uint8 va_tag = VARTAG_EXTERNAL(tp + ctx->offset); - if (va_tag != VARTAG_ONDISK) + if (va_tag != VARTAG_ONDISK_OID) { report_corruption(ctx, psprintf("toasted attribute has unexpected TOAST tag %u", @@ -1774,28 +1776,28 @@ check_tuple_attribute(HeapCheckContext *ctx) return true; /* It is external, and we're looking at a page on disk */ - toast_pointer_valueid = toast_pointer.va_valueid; /* * Must copy attr into toast_pointer for alignment considerations */ - VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + toast_external_info_get_data(attr, &toast_pointer); + toast_pointer_valueid = toast_pointer.value; /* Toasted attributes too large to be untoasted should never be stored */ - if (toast_pointer.va_rawsize > VARLENA_SIZE_LIMIT) + if (toast_pointer.rawsize > VARLENA_SIZE_LIMIT) report_corruption(ctx, psprintf("toast value %" PRIu64 " rawsize %d exceeds limit %d", toast_pointer_valueid, - toast_pointer.va_rawsize, + toast_pointer.rawsize, VARLENA_SIZE_LIMIT)); - if (VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer)) + if (TOAST_EXTERNAL_IS_COMPRESSED(toast_pointer)) { ToastCompressionId cmid; bool valid = false; /* Compressed attributes should have a valid compression method */ - cmid = TOAST_COMPRESS_METHOD(&toast_pointer); + cmid = toast_pointer.compression_method; switch (cmid) { /* List of all valid compression method IDs */ @@ -1849,7 +1851,8 @@ check_tuple_attribute(HeapCheckContext *ctx) ta = (ToastedAttribute *) palloc0(sizeof(ToastedAttribute)); - VARATT_EXTERNAL_GET_POINTER(ta->toast_pointer, attr); + toast_external_info_get_data(attr, &ta->toast_pointer); + ta->info = toast_external_get_info(VARTAG_EXTERNAL(attr)); ta->blkno = ctx->blkno; ta->offnum = ctx->offnum; ta->attnum = ctx->attnum; @@ -1876,9 +1879,11 @@ check_toasted_attribute(HeapCheckContext *ctx, ToastedAttribute *ta) int32 expected_chunk_seq = 0; int32 last_chunk_seq; uint64 toast_valueid; - int32 max_chunk_size = TOAST_MAX_CHUNK_SIZE; + int32 max_chunk_size; - extsize = VARATT_EXTERNAL_GET_EXTSIZE(ta->toast_pointer); + extsize = ta->toast_pointer.extsize; + + max_chunk_size = ta->info->maximum_chunk_size; last_chunk_seq = (extsize - 1) / max_chunk_size; /* @@ -1887,7 +1892,7 @@ check_toasted_attribute(HeapCheckContext *ctx, ToastedAttribute *ta) ScanKeyInit(&toastkey, (AttrNumber) 1, BTEqualStrategyNumber, F_OIDEQ, - ObjectIdGetDatum(ta->toast_pointer.va_valueid)); + ObjectIdGetDatum(ta->toast_pointer.value)); /* * Check if any chunks for this toasted object exist in the toast table, @@ -1907,7 +1912,7 @@ check_toasted_attribute(HeapCheckContext *ctx, ToastedAttribute *ta) } systable_endscan_ordered(toastscan); - toast_valueid = ta->toast_pointer.va_valueid; + toast_valueid = ta->toast_pointer.value; if (!found_toasttup) report_toast_corruption(ctx, ta, diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 114bdafafdfa..3b98606a1701 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -4137,6 +4137,8 @@ timeout_params timerCA tlist_vinfo toast_compress_header +toast_external_data +toast_external_info tokenize_error_callback_arg transferMode transfer_thread_arg -- 2.50.0