From b0a266175e5a244556d6d0d5aa365e460d46e1b6 Mon Sep 17 00:00:00 2001 From: Hannu Krosing Date: Sun, 23 Aug 2026 20:08:55 +0000 Subject: [PATCH v4 2/9] Add Direct TOAST catalog, GUC, and reloptions infrastructure Introduce base structures, macros, and configuration options for Direct TOAST: - Add struct varatt_direct and VARTAG_DIRECT (tag 19) in varatt.h. - Add toast_flavour reloption ('plain' / 'direct') in reloptions.c. - Add default_toast_flavour GUC in guc_tables.c. - Update create_toast_table() to support chunk_tids and chunk_tid_offsets columns for direct TOAST tables, with partial B-Tree indexing on chunk_id. --- src/backend/access/common/reloptions.c | 20 +++++++ src/backend/catalog/toasting.c | 26 ++++++++- src/backend/utils/misc/guc_tables.c | 7 +++ src/backend/utils/misc/postgresql.conf.sample | 1 + src/include/access/toast_internals.h | 9 ++++ src/include/utils/rel.h | 8 +++ src/include/varatt.h | 53 ++++++++++++++++++- 7 files changed, 121 insertions(+), 3 deletions(-) diff --git a/src/backend/access/common/reloptions.c b/src/backend/access/common/reloptions.c index ea9a0417909..02db33016a2 100644 --- a/src/backend/access/common/reloptions.c +++ b/src/backend/access/common/reloptions.c @@ -530,6 +530,13 @@ static relopt_enum_elt_def StdRdOptIndexCleanupValues[] = {(const char *) NULL} /* list terminator */ }; +static relopt_enum_elt_def toastFlavourOptValues[] = +{ + {"plain", TOAST_FLAVOUR_PLAIN}, + {"direct", TOAST_FLAVOUR_DIRECT}, + {(const char *) NULL} /* list terminator */ +}; + /* values from GistOptBufferingMode */ static relopt_enum_elt_def gistBufferingOptValues[] = { @@ -581,6 +588,17 @@ static relopt_enum enumRelOpts[] = STDRD_OPTION_TOAST_VALUE_TYPE_OID, gettext_noop("Valid values are \"oid\" and \"oid8\".") }, + { + { + "toast_flavour", + "Sets the TOAST flavour for this table", + RELOPT_KIND_HEAP, + ShareUpdateExclusiveLock + }, + toastFlavourOptValues, + TOAST_FLAVOUR_NOT_SET, + gettext_noop("Valid values are \"plain\" and \"direct\".") + }, { { "buffering", @@ -2114,6 +2132,8 @@ static const relopt_parse_elt stdRdOptionsTab[] = { offsetof(StdRdOptions, parallel_workers)}, {"vacuum_index_cleanup", RELOPT_TYPE_ENUM, offsetof(StdRdOptions, vacuum_index_cleanup)}, + {"toast_flavour", RELOPT_TYPE_ENUM, + offsetof(StdRdOptions, toast_flavour)}, {"vacuum_truncate", RELOPT_TYPE_TERNARY, offsetof(StdRdOptions, vacuum_truncate)}, {"vacuum_max_eager_freeze_failure_rate", RELOPT_TYPE_REAL, diff --git a/src/backend/catalog/toasting.c b/src/backend/catalog/toasting.c index 3058ec14223..13da87fdcbd 100644 --- a/src/backend/catalog/toasting.c +++ b/src/backend/catalog/toasting.c @@ -254,7 +254,7 @@ create_toast_table(Relation rel, Oid toastOid, Oid toastIndexOid, } /* this is pretty painful... need a tuple descriptor */ - tupdesc = CreateTemplateTupleDesc(3); + tupdesc = CreateTemplateTupleDesc(5); TupleDescInitEntry(tupdesc, (AttrNumber) 1, "chunk_id", toast_chunkid_typid, @@ -267,6 +267,14 @@ create_toast_table(Relation rel, Oid toastOid, Oid toastIndexOid, "chunk_data", BYTEAOID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 4, + "chunk_tids", + TIDARRAYOID, + -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 5, + "chunk_tid_offsets", + INT8ARRAYOID, + -1, 0); /* * Ensure that the toast table doesn't itself get toasted, or we'll be @@ -276,15 +284,21 @@ create_toast_table(Relation rel, Oid toastOid, Oid toastIndexOid, TupleDescAttr(tupdesc, 0)->attstorage = TYPSTORAGE_PLAIN; TupleDescAttr(tupdesc, 1)->attstorage = TYPSTORAGE_PLAIN; TupleDescAttr(tupdesc, 2)->attstorage = TYPSTORAGE_PLAIN; + TupleDescAttr(tupdesc, 3)->attstorage = TYPSTORAGE_PLAIN; + TupleDescAttr(tupdesc, 4)->attstorage = TYPSTORAGE_PLAIN; /* Toast field should not be compressed */ TupleDescAttr(tupdesc, 0)->attcompression = InvalidCompressionMethod; TupleDescAttr(tupdesc, 1)->attcompression = InvalidCompressionMethod; TupleDescAttr(tupdesc, 2)->attcompression = InvalidCompressionMethod; + TupleDescAttr(tupdesc, 3)->attcompression = InvalidCompressionMethod; + TupleDescAttr(tupdesc, 4)->attcompression = InvalidCompressionMethod; populate_compact_attribute(tupdesc, 0); populate_compact_attribute(tupdesc, 1); populate_compact_attribute(tupdesc, 2); + populate_compact_attribute(tupdesc, 3); + populate_compact_attribute(tupdesc, 4); TupleDescFinalize(tupdesc); @@ -351,7 +365,15 @@ create_toast_table(Relation rel, Oid toastOid, Oid toastIndexOid, indexInfo->ii_IndexAttrNumbers[1] = 2; indexInfo->ii_Expressions = NIL; indexInfo->ii_ExpressionsState = NIL; - indexInfo->ii_Predicate = NIL; + { + NullTest *ntest = makeNode(NullTest); + + ntest->arg = (Expr *) makeVar(1, 1, OIDOID, -1, InvalidOid, 0); + ntest->nulltesttype = IS_NOT_NULL; + ntest->argisrow = false; + ntest->location = -1; + indexInfo->ii_Predicate = list_make1(ntest); + } indexInfo->ii_PredicateState = NULL; indexInfo->ii_ExclusionOps = NULL; indexInfo->ii_ExclusionProcs = NULL; diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 342aaeef59a..11d8b71f0a5 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -33,6 +33,7 @@ #include "access/gin.h" #include "access/slru.h" #include "access/toast_compression.h" +#include "access/toast_internals.h" #include "access/twophase.h" #include "access/xlog_internal.h" #include "access/xlogprefetcher.h" @@ -287,6 +288,12 @@ static const struct config_enum_entry xmloption_options[] = { StaticAssertDecl(lengthof(xmloption_options) == (XMLOPTION_CONTENT + 2), "array length mismatch"); +static const struct config_enum_entry toast_flavour_options[] = { + {"plain", TOAST_FLAVOUR_PLAIN, false}, + {"direct", TOAST_FLAVOUR_DIRECT, false}, + {NULL, 0, false} +}; + /* * Although only "on", "off", and "safe_encoding" are documented, we * accept all the likely variants of "on" and "off". diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index e759f06b50f..09ebb2f5445 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -807,6 +807,7 @@ #default_table_access_method = 'heap' #default_tablespace = '' # a tablespace name, '' uses the default #default_toast_compression = pglz # pglz or lz4 +#toast_flavour = plain # plain or direct #temp_tablespaces = '' # a list of tablespace names, '' uses # only default tablespace #check_function_bodies = on diff --git a/src/include/access/toast_internals.h b/src/include/access/toast_internals.h index e03cc1204be..15584781280 100644 --- a/src/include/access/toast_internals.h +++ b/src/include/access/toast_internals.h @@ -17,6 +17,15 @@ #include "storage/lockdefs.h" #include "utils/relcache.h" #include "utils/snapshot.h" +#include "utils/rel.h" + +#define RelationGetToastFlavour(relation) \ + ((relation)->rd_options ? \ + (((StdRdOptions *) (relation)->rd_options)->toast_flavour != TOAST_FLAVOUR_NOT_SET ? \ + ((StdRdOptions *) (relation)->rd_options)->toast_flavour : (ToastFlavour) toast_flavour) \ + : (ToastFlavour) toast_flavour) + +extern PGDLLIMPORT int toast_flavour; /* * The information at the start of the compressed toast data. diff --git a/src/include/utils/rel.h b/src/include/utils/rel.h index 9c8d17337b8..e7db822e170 100644 --- a/src/include/utils/rel.h +++ b/src/include/utils/rel.h @@ -358,6 +358,13 @@ typedef enum StdRdOptToastValueType STDRD_OPTION_TOAST_VALUE_TYPE_OID8, } StdRdOptToastValueType; +typedef enum ToastFlavour +{ + TOAST_FLAVOUR_NOT_SET = -1, + TOAST_FLAVOUR_PLAIN = 0, + TOAST_FLAVOUR_DIRECT = 1 +} ToastFlavour; + typedef struct StdRdOptions { int32 vl_len_; /* varlena header (do not touch directly!) */ @@ -376,6 +383,7 @@ typedef struct StdRdOptions * to freeze. 0 if disabled, -1 if unspecified. */ double vacuum_max_eager_freeze_failure_rate; + ToastFlavour toast_flavour; } StdRdOptions; #define HEAP_MIN_FILLFACTOR 10 diff --git a/src/include/varatt.h b/src/include/varatt.h index 77041f7673a..c0d1a8444a2 100644 --- a/src/include/varatt.h +++ b/src/include/varatt.h @@ -15,6 +15,8 @@ #ifndef VARATT_H #define VARATT_H +#include "storage/itemptr.h" + /* * varatt_external_oid is a traditional "TOAST pointer", that is, the * information needed to fetch a Datum stored out-of-line in a TOAST table. @@ -80,6 +82,15 @@ VARATT_EXTERNAL_OID8_SET_VALUEID(varatt_external_oid8 *toast_pointer, Oid8 id) toast_pointer->va_valueid_hi = (uint32) (id >> 32); } +typedef struct varatt_direct +{ + int32 va_rawsize; /* Original data size (includes header) */ + uint32 va_extinfo; /* External saved size (without header) and + * compression method */ + Oid va_toastrelid; /* RelID of TOAST table containing it */ + ItemPointerData va_tid; /* Physical TID of the final chunk */ +} varatt_direct; + /* * These macros define the "saved size" portion of va_extinfo. Its remaining * two high-order bits identify the compression method. @@ -130,7 +141,8 @@ typedef enum vartag_external VARTAG_EXPANDED_RO = 2, VARTAG_EXPANDED_RW = 3, VARTAG_ONDISK_OID8 = 4, - VARTAG_ONDISK_OID = 18 + VARTAG_ONDISK_OID = 18, + VARTAG_DIRECT = 19 } vartag_external; /* Is a TOAST pointer either type of expanded-object pointer? */ @@ -153,6 +165,8 @@ VARTAG_SIZE(vartag_external tag) return sizeof(varatt_external_oid); else if (tag == VARTAG_ONDISK_OID8) return sizeof(varatt_external_oid8); + else if (tag == VARTAG_DIRECT) + return sizeof(varatt_direct); else { Assert(false); @@ -414,6 +428,24 @@ VARATT_IS_EXTERNAL_ONDISK(const void *PTR) return (tag == VARTAG_ONDISK_OID || tag == VARTAG_ONDISK_OID8); } +static inline bool +VARATT_IS_EXTERNAL_DIRECT(const void *PTR) +{ + return VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_DIRECT; +} + +static inline bool +VARATT_IS_EXTERNAL_ONDISK_OR_DIRECT(const void *PTR) +{ + vartag_external tag; + + if (!VARATT_IS_EXTERNAL(PTR)) + return false; + tag = VARTAG_EXTERNAL(PTR); + return (tag == VARTAG_ONDISK_OID || tag == VARTAG_ONDISK_OID8 || + tag == VARTAG_DIRECT); +} + /* Is varlena datum an indirect pointer? */ static inline bool VARATT_IS_EXTERNAL_INDIRECT(const void *PTR) @@ -581,4 +613,23 @@ VARATT_EXTINFO_IS_COMPRESSED(uint32 extinfo, int32 rawsize) return VARATT_EXTINFO_GET_EXTSIZE(extinfo) < (Size) (rawsize - VARHDRSZ); } +static inline Size +VARATT_DIRECT_GET_EXTSIZE(varatt_direct toast_pointer) +{ + return toast_pointer.va_extinfo & VARLENA_EXTSIZE_MASK; +} + +static inline uint32 +VARATT_DIRECT_GET_COMPRESS_METHOD(varatt_direct toast_pointer) +{ + return toast_pointer.va_extinfo >> VARLENA_EXTSIZE_BITS; +} + +static inline bool +VARATT_DIRECT_IS_COMPRESSED(varatt_direct toast_pointer) +{ + return VARATT_DIRECT_GET_EXTSIZE(toast_pointer) < + (Size) (toast_pointer.va_rawsize - VARHDRSZ); +} + #endif -- 2.55.0.1082.g2b9226bbc0-goog