From 18e2f0958672c95afa7a4b8f6282a708462ea8d0 Mon Sep 17 00:00:00 2001 From: Pavel Borisov Date: Mon, 21 Sep 2026 19:15:15 +0400 Subject: [PATCH v1] Halve peak memory allocation for toast_flatten_tuple Existing code detoasts every external attribute into a Datum array, then calls heap_fill_tuple on the complete set. So around 2x tuple length memory is allocated (result tuple plus all detoasted values) Mainly this affects logical replication with REPLICA IDENTITY FULL with tables with several large toasted columns: every UPDATE/DELETE flattens the old tuple. The other callers -- catalog cache loading and PL/pgSQL record assignment -- handle smaller tuples where the difference is negligible. Flatten TOASTed tuples with a two-stage approach: - For each external attribute build a VARHDRSZ stub whose header matches what detoast_external_attr will return (size and compression tag). Pass the stubs through heap_fill_tuple_attr with skip_copy=true so alignment and sizes are computed without copying data. attr_data[] records the destination pointer for every non-null attribute. - Detoast one attribute at a time into the pre-computed position, pfree immediately. Peak memory usage decreased from 2x tuple to 1x tuple plus one detoasted value. For compressed external datums detoast_external_attr returns the value still compressed, so the stub carries the compressed size and SET_VARSIZE_COMPRESSED; using toast_raw_datum_size (uncompressed) would produce a layout/data size mismatch. Short-header varlena packing is applied the same way fill_val does it, keeping the result byte-identical to the old code path. INDIRECT externals can wrap compressed or short-header datums whose detoasted format the stub cannot predict. Detoast them inline as previously. The extra per-attribute cost (one VARHDRSZ palloc/pfree for the stub, toast pointer decode for compression detection) is negligible compared the toast-fetch I/O that dominates every call. --- src/backend/access/common/heaptuple.c | 127 +++++++++++++++++---- src/backend/access/heap/heaptoast.c | 158 +++++++++++++++++++++++--- src/include/access/htup_details.h | 6 + src/test/subscription/t/100_bugs.pl | 75 ++++++++++++ 4 files changed, 326 insertions(+), 40 deletions(-) diff --git a/src/backend/access/common/heaptuple.c b/src/backend/access/common/heaptuple.c index f30346469ed..3b8b8d5e437 100644 --- a/src/backend/access/common/heaptuple.c +++ b/src/backend/access/common/heaptuple.c @@ -267,22 +267,31 @@ heap_compute_data_size(TupleDesc tupleDesc, } /* - * Per-attribute helper for heap_fill_tuple and other routines building tuples. + * Per-attribute helper for heap_fill_tuple_attr. * - * Fill in either a data value or a bit in the null bitmask + * Fill in either a data value or a bit in the null bitmask. + * + * When skip_copy is true the data pointer is advanced without writing + * anything. If attr_data_out is non-NULL, it receives the aligned position + * where the attribute's payload starts (NULL for null attrs). */ static inline void -fill_val(CompactAttribute *att, - uint8 **bit, - int *bitmask, - char **dataP, - uint16 *infomask, - Datum datum, - bool isnull) +fill_val_attr(CompactAttribute *att, + uint8 **bit, + int *bitmask, + char **dataP, + uint16 *infomask, + Datum datum, + bool isnull, + bool skip_copy, + char **attr_data_out) { Size data_length; char *data = *dataP; + if (attr_data_out) + *attr_data_out = NULL; + /* * If we're building a null bitmap, set the appropriate bit for the * current column value here. @@ -315,7 +324,10 @@ fill_val(CompactAttribute *att, { /* pass-by-value */ data = (char *) att_nominal_alignby(data, att->attalignby); - store_att_byval(data, datum, att->attlen); + if (attr_data_out) + *attr_data_out = data; + if (!skip_copy) + store_att_byval(data, datum, att->attlen); data_length = att->attlen; } else if (att->attlen == -1) @@ -335,36 +347,53 @@ fill_val(CompactAttribute *att, ExpandedObjectHeader *eoh = DatumGetEOHP(datum); data = (char *) att_nominal_alignby(data, att->attalignby); + if (attr_data_out) + *attr_data_out = data; data_length = EOH_get_flat_size(eoh); - EOH_flatten_into(eoh, data, data_length); + if (!skip_copy) + EOH_flatten_into(eoh, data, data_length); } else { + if (attr_data_out) + *attr_data_out = data; *infomask |= HEAP_HASEXTERNAL; /* no alignment, since it's short by definition */ data_length = VARSIZE_EXTERNAL(val); - memcpy(data, val, data_length); + if (!skip_copy) + memcpy(data, val, data_length); } } else if (VARATT_IS_SHORT(val)) { /* no alignment for short varlenas */ + if (attr_data_out) + *attr_data_out = data; data_length = VARSIZE_SHORT(val); - memcpy(data, val, data_length); + if (!skip_copy) + memcpy(data, val, data_length); } else if (att->attispackable && VARATT_CAN_MAKE_SHORT(val)) { /* convert to short varlena -- no alignment */ + if (attr_data_out) + *attr_data_out = data; data_length = VARATT_CONVERTED_SHORT_SIZE(val); - SET_VARSIZE_SHORT(data, data_length); - memcpy(data + 1, VARDATA(val), data_length - 1); + if (!skip_copy) + { + SET_VARSIZE_SHORT(data, data_length); + memcpy(data + 1, VARDATA(val), data_length - 1); + } } else { /* full 4-byte header varlena */ data = (char *) att_nominal_alignby(data, att->attalignby); + if (attr_data_out) + *attr_data_out = data; data_length = VARSIZE(val); - memcpy(data, val, data_length); + if (!skip_copy) + memcpy(data, val, data_length); } } else if (att->attlen == -2) @@ -372,22 +401,46 @@ fill_val(CompactAttribute *att, /* cstring ... never needs alignment */ *infomask |= HEAP_HASVARWIDTH; Assert(att->attalignby == sizeof(char)); + if (attr_data_out) + *attr_data_out = data; data_length = strlen(DatumGetCString(datum)) + 1; - memcpy(data, DatumGetPointer(datum), data_length); + if (!skip_copy) + memcpy(data, DatumGetPointer(datum), data_length); } else { /* fixed-length pass-by-reference */ data = (char *) att_nominal_alignby(data, att->attalignby); + if (attr_data_out) + *attr_data_out = data; Assert(att->attlen > 0); data_length = att->attlen; - memcpy(data, DatumGetPointer(datum), data_length); + if (!skip_copy) + memcpy(data, DatumGetPointer(datum), data_length); } data += data_length; *dataP = data; } +/* + * Per-attribute helper for heap_fill_tuple and other routines building tuples. + * + * Fill in either a data value or a bit in the null bitmask + */ +static inline void +fill_val(CompactAttribute *att, + uint8 **bit, + int *bitmask, + char **dataP, + uint16 *infomask, + Datum datum, + bool isnull) +{ + fill_val_attr(att, bit, bitmask, dataP, infomask, datum, isnull, + false, NULL); +} + /* * heap_fill_tuple * Load data portion of a tuple from values/isnull arrays @@ -402,6 +455,28 @@ heap_fill_tuple(TupleDesc tupleDesc, const Datum *values, const bool *isnull, char *data, Size data_size, uint16 *infomask, uint8 *bit) +{ + heap_fill_tuple_attr(tupleDesc, values, isnull, + data, data_size, infomask, bit, + NULL, NULL); +} + +/* + * heap_fill_tuple_attr + * Like heap_fill_tuple, but with per-attribute skip and position output. + * + * skip_copy: when entry is true, advance past the attribute without copying. + * attr_data: receives the aligned data position of each non-null attribute. + * + * Either pointer may be NULL, giving the same behavior as heap_fill_tuple. + */ +void +heap_fill_tuple_attr(TupleDesc tupleDesc, + const Datum *values, const bool *isnull, + char *data, Size data_size, + uint16 *infomask, uint8 *bit, + const bool *skip_copy, + char **attr_data) { uint8 *bitP; int bitmask; @@ -430,13 +505,15 @@ heap_fill_tuple(TupleDesc tupleDesc, { CompactAttribute *attr = TupleDescCompactAttr(tupleDesc, i); - fill_val(attr, - bitP ? &bitP : NULL, - &bitmask, - &data, - infomask, - values ? values[i] : PointerGetDatum(NULL), - isnull ? isnull[i] : true); + fill_val_attr(attr, + bitP ? &bitP : NULL, + &bitmask, + &data, + infomask, + values ? values[i] : PointerGetDatum(NULL), + isnull ? isnull[i] : true, + skip_copy ? skip_copy[i] : false, + attr_data ? &attr_data[i] : NULL); } Assert((data - start) == data_size); diff --git a/src/backend/access/heap/heaptoast.c b/src/backend/access/heap/heaptoast.c index f8a54ff2582..650e2be19ad 100644 --- a/src/backend/access/heap/heaptoast.c +++ b/src/backend/access/heap/heaptoast.c @@ -30,6 +30,7 @@ #include "access/heaptoast.h" #include "access/toast_helper.h" #include "access/toast_internals.h" +#include "utils/expandeddatum.h" #include "utils/fmgroids.h" @@ -344,17 +345,31 @@ heap_toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, * * Note: we expect the caller already checked HeapTupleHasExternal(tup), * so there is no need for a short-circuit path. + * + * External attributes are detoasted one at a time into pre-computed + * positions in the result tuple, so peak memory is the result tuple + * plus one detoasted attribute rather than all of them at once. * ---------- */ HeapTuple toast_flatten_tuple(HeapTuple tup, TupleDesc tupleDesc) { HeapTuple new_tuple; + HeapTupleHeader td; int numAttrs = tupleDesc->natts; int i; Datum toast_values[MaxTupleAttributeNumber]; bool toast_isnull[MaxTupleAttributeNumber]; + bool toast_skip[MaxTupleAttributeNumber]; bool toast_free[MaxTupleAttributeNumber]; + char *toast_attr_data[MaxTupleAttributeNumber]; + Datum toast_origvals[MaxTupleAttributeNumber]; + int toasted_atts[MaxTupleAttributeNumber]; + int ntoasted = 0; + bool hasnull = false; + Size data_len; + int len; + int hoff; /* * Break down the tuple into fields. @@ -362,31 +377,144 @@ toast_flatten_tuple(HeapTuple tup, TupleDesc tupleDesc) Assert(numAttrs <= MaxTupleAttributeNumber); heap_deform_tuple(tup, tupleDesc, toast_values, toast_isnull); - memset(toast_free, 0, numAttrs * sizeof(bool)); + memset(toast_skip, false, numAttrs * sizeof(bool)); + memset(toast_free, false, numAttrs * sizeof(bool)); + /* + * Replace each external toast pointer with a size-only stub that has the + * correct VARSIZE header. heap_compute_data_size and + * heap_fill_tuple_attr read only VARSIZE from these, never the payload. + */ for (i = 0; i < numAttrs; i++) { + varlena *val; + varlena *stub; + + if (toast_isnull[i] || TupleDescCompactAttr(tupleDesc, i)->attlen != -1) + continue; + + val = (varlena *) DatumGetPointer(toast_values[i]); + if (!VARATT_IS_EXTERNAL(val)) + continue; + /* - * Look at non-null varlena attributes + * INDIRECT externals can wrap compressed or short-header datums whose + * detoasted format the stub cannot predict. Detoast inline. */ - if (!toast_isnull[i] && TupleDescCompactAttr(tupleDesc, i)->attlen == -1) + if (VARATT_IS_EXTERNAL_INDIRECT(val)) { - varlena *new_value; + toast_values[i] = PointerGetDatum(detoast_external_attr(val)); + toast_free[i] = true; + continue; + } - new_value = (varlena *) DatumGetPointer(toast_values[i]); - if (VARATT_IS_EXTERNAL(new_value)) - { - new_value = detoast_external_attr(new_value); - toast_values[i] = PointerGetDatum(new_value); - toast_free[i] = true; - } + /* + * Build a stub whose VARSIZE matches what detoast_external_attr will + * return. For on-disk compressed datums the fetched value stays + * compressed, so the stub carries the compressed size and tag. For + * expanded objects EOH_get_flat_size gives the 4B_U size that + * flattening will produce. + */ + stub = (varlena *) palloc(VARHDRSZ); + if (VARATT_IS_EXTERNAL_ONDISK(val)) + { + toast_external_data toast_ext_data; + int32 extsize; + + toast_external_info_get(val, &toast_ext_data); + extsize = VARATT_EXTINFO_GET_EXTSIZE(toast_ext_data.extinfo); + if (VARATT_EXTINFO_IS_COMPRESSED(toast_ext_data.extinfo, + toast_ext_data.rawsize)) + SET_VARSIZE_COMPRESSED(stub, extsize + VARHDRSZ); + else + SET_VARSIZE(stub, extsize + VARHDRSZ); + } + else + { + Assert(VARATT_IS_EXTERNAL_EXPANDED(val)); + SET_VARSIZE(stub, EOH_get_flat_size(DatumGetEOHP(toast_values[i]))); } + + toast_origvals[ntoasted] = toast_values[i]; + toasted_atts[ntoasted] = i; + ntoasted++; + + toast_values[i] = PointerGetDatum(stub); + toast_skip[i] = true; } - /* - * Form the reconfigured tuple. - */ - new_tuple = heap_form_tuple(tupleDesc, toast_values, toast_isnull); + /* Compute tuple header and data lengths */ + for (i = 0; i < numAttrs; i++) + { + if (toast_isnull[i]) + { + hasnull = true; + break; + } + } + + len = offsetof(HeapTupleHeaderData, t_bits); + if (hasnull) + len += BITMAPLEN(numAttrs); + hoff = len = MAXALIGN(len); + + data_len = heap_compute_data_size(tupleDesc, toast_values, toast_isnull); + len += data_len; + + new_tuple = (HeapTuple) palloc0(HEAPTUPLESIZE + len); + new_tuple->t_data = td = (HeapTupleHeader) ((char *) new_tuple + HEAPTUPLESIZE); + + new_tuple->t_len = len; + ItemPointerSetInvalid(&(new_tuple->t_self)); + new_tuple->t_tableOid = InvalidOid; + + HeapTupleHeaderSetDatumLength(td, len); + HeapTupleHeaderSetTypeId(td, tupleDesc->tdtypeid); + HeapTupleHeaderSetTypMod(td, tupleDesc->tdtypmod); + ItemPointerSetInvalid(&(td->t_ctid)); + + HeapTupleHeaderSetNatts(td, numAttrs); + td->t_hoff = hoff; + + /* Fill non-toasted attributes, recording positions of toasted ones */ + heap_fill_tuple_attr(tupleDesc, + toast_values, toast_isnull, + (char *) td + hoff, + data_len, + &td->t_infomask, + (hasnull ? td->t_bits : NULL), + toast_skip, + toast_attr_data); + + /* Free stubs before detoasting to keep peak memory low */ + for (i = 0; i < ntoasted; i++) + pfree(DatumGetPointer(toast_values[toasted_atts[i]])); + + /* Detoast one attribute at a time into the pre-computed position */ + for (i = 0; i < ntoasted; i++) + { + int attn = toasted_atts[i]; + CompactAttribute *att = TupleDescCompactAttr(tupleDesc, attn); + varlena *detoasted; + char *dest = toast_attr_data[attn]; + + detoasted = detoast_external_attr( + (varlena *) DatumGetPointer(toast_origvals[i])); + + if (att->attispackable && VARATT_CAN_MAKE_SHORT(detoasted)) + { + Size short_len = VARATT_CONVERTED_SHORT_SIZE(detoasted); + + SET_VARSIZE_SHORT(dest, short_len); + memcpy(dest + 1, VARDATA(detoasted), short_len - 1); + } + else + { + memcpy(dest, detoasted, VARSIZE(detoasted)); + } + + pfree(detoasted); + } /* * Be sure to copy the tuple's identity fields. We also make a point of diff --git a/src/include/access/htup_details.h b/src/include/access/htup_details.h index 77a6c48fd71..e1a1b22ac87 100644 --- a/src/include/access/htup_details.h +++ b/src/include/access/htup_details.h @@ -798,6 +798,12 @@ extern void heap_fill_tuple(TupleDesc tupleDesc, const Datum *values, const bool *isnull, char *data, Size data_size, uint16 *infomask, uint8 *bit); +extern void heap_fill_tuple_attr(TupleDesc tupleDesc, + const Datum *values, const bool *isnull, + char *data, Size data_size, + uint16 *infomask, uint8 *bit, + const bool *skip_copy, + char **attr_data); extern bool heap_attisnull(HeapTuple tup, int attnum, TupleDesc tupleDesc); extern Datum nocachegetattr(HeapTuple tup, int attnum, TupleDesc tupleDesc); diff --git a/src/test/subscription/t/100_bugs.pl b/src/test/subscription/t/100_bugs.pl index 06c032a8e64..35f9cc61f22 100644 --- a/src/test/subscription/t/100_bugs.pl +++ b/src/test/subscription/t/100_bugs.pl @@ -759,4 +759,79 @@ DROP SUBSCRIPTION sub_drop_refresh; $node_publisher->stop('fast'); $node_subscriber->stop('fast'); +# Test that toast_flatten_tuple correctly handles both compressed and +# uncompressed external toast values with REPLICA IDENTITY FULL. +# Compressed external datums keep their compressed format after +# detoast_external_attr, so the stub size must match. + +$node_publisher->rotate_logfile(); +$node_subscriber->rotate_logfile(); +$node_publisher->start; +$node_subscriber->start; + +$publisher_connstr = $node_publisher->connstr . ' dbname=postgres'; + +# Use a compressible column (text, default EXTENDED storage) and an +# incompressible one (bytea forced to EXTERNAL) to cover both paths. +$node_publisher->safe_psql( + 'postgres', qq{ +CREATE TABLE tab_toast_flatten ( + a int PRIMARY KEY, + b text, + c bytea); +ALTER TABLE tab_toast_flatten ALTER COLUMN c SET STORAGE EXTERNAL; +ALTER TABLE tab_toast_flatten REPLICA IDENTITY FULL; +}); + +$node_subscriber->safe_psql( + 'postgres', qq{ +CREATE TABLE tab_toast_flatten ( + a int PRIMARY KEY, + b text, + c bytea); +ALTER TABLE tab_toast_flatten ALTER COLUMN c SET STORAGE EXTERNAL; +ALTER TABLE tab_toast_flatten REPLICA IDENTITY FULL; +}); + +$node_publisher->safe_psql('postgres', + "CREATE PUBLICATION pub_toast FOR TABLE tab_toast_flatten"); +$node_subscriber->safe_psql('postgres', + "CREATE SUBSCRIPTION sub_toast CONNECTION '$publisher_connstr' PUBLICATION pub_toast WITH (copy_data = false)" +); +$node_subscriber->wait_for_subscription_sync($node_publisher, 'sub_toast'); + +# Insert rows with large values that will be toasted. repeat() produces +# highly compressible data (compressed external); random bytea will not +# compress and goes to external storage uncompressed. +$node_publisher->safe_psql( + 'postgres', qq{ +INSERT INTO tab_toast_flatten VALUES ( + 1, + repeat('x', 100000), + decode(repeat('ab', 50000), 'hex')); +}); +$node_publisher->wait_for_catchup('sub_toast'); + +$result = $node_subscriber->safe_psql('postgres', + "SELECT a, length(b), length(c) FROM tab_toast_flatten WHERE a = 1"); +is($result, '1|100000|50000', + 'toast_flatten_tuple: insert with toasted values'); + +# Update forces the old tuple to be flattened for the change message. +$node_publisher->safe_psql('postgres', + "UPDATE tab_toast_flatten SET a = 2 WHERE a = 1"); +$node_publisher->wait_for_catchup('sub_toast'); + +$result = $node_subscriber->safe_psql('postgres', + "SELECT a, length(b), length(c) FROM tab_toast_flatten WHERE a = 2"); +is($result, '2|100000|50000', + 'toast_flatten_tuple: update with compressed and uncompressed toasted values'); + +$node_subscriber->safe_psql('postgres', "DROP SUBSCRIPTION sub_toast"); +$node_publisher->safe_psql('postgres', "DROP PUBLICATION pub_toast"); +$node_publisher->safe_psql('postgres', "DROP TABLE tab_toast_flatten"); + +$node_publisher->stop('fast'); +$node_subscriber->stop('fast'); + done_testing(); -- 2.50.1 (Apple Git-155)