From aea486a6b3aa8dee840fe7be83421963f83a6f67 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Thu, 19 Jun 2025 09:57:19 +0900 Subject: [PATCH v2 01/13] Refactor some TOAST value ID code to use uint64 instead of Oid This change is a mechanical switch to change most of the code paths that assume TOAST value IDs to be Oids to become uint64, easing an upcoming change to allow 8-byte TOAST values. The areas touched are related to table AM, amcheck and logical decoding's reorder buffer. A good chunk of the changes involve switching printf() markers from %u to PRIu64. --- src/include/access/heaptoast.h | 2 +- src/include/access/tableam.h | 4 +- src/backend/access/common/toast_internals.c | 8 +-- src/backend/access/heap/heaptoast.c | 12 ++-- .../replication/logical/reorderbuffer.c | 14 ++-- contrib/amcheck/verify_heapam.c | 69 +++++++++++-------- 6 files changed, 62 insertions(+), 47 deletions(-) diff --git a/src/include/access/heaptoast.h b/src/include/access/heaptoast.h index 6385a27caf83..6e3558cbd6d2 100644 --- a/src/include/access/heaptoast.h +++ b/src/include/access/heaptoast.h @@ -142,7 +142,7 @@ extern HeapTuple toast_build_flattened_tuple(TupleDesc tupleDesc, * Fetch a slice from a toast value stored in a heap table. * ---------- */ -extern void heap_fetch_toast_slice(Relation toastrel, Oid valueid, +extern void heap_fetch_toast_slice(Relation toastrel, uint64 valueid, int32 attrsize, int32 sliceoffset, int32 slicelength, struct varlena *result); diff --git a/src/include/access/tableam.h b/src/include/access/tableam.h index 1c9e802a6b12..b640047d2fdc 100644 --- a/src/include/access/tableam.h +++ b/src/include/access/tableam.h @@ -740,7 +740,7 @@ typedef struct TableAmRoutine * table implemented by this AM. See table_relation_fetch_toast_slice() * for more details. */ - void (*relation_fetch_toast_slice) (Relation toastrel, Oid valueid, + void (*relation_fetch_toast_slice) (Relation toastrel, uint64 valueid, int32 attrsize, int32 sliceoffset, int32 slicelength, @@ -1873,7 +1873,7 @@ table_relation_toast_am(Relation rel) * stored. */ static inline void -table_relation_fetch_toast_slice(Relation toastrel, Oid valueid, +table_relation_fetch_toast_slice(Relation toastrel, uint64 valueid, int32 attrsize, int32 sliceoffset, int32 slicelength, struct varlena *result) { diff --git a/src/backend/access/common/toast_internals.c b/src/backend/access/common/toast_internals.c index 7d8be8346ce5..4a1342da6e1b 100644 --- a/src/backend/access/common/toast_internals.c +++ b/src/backend/access/common/toast_internals.c @@ -26,8 +26,8 @@ #include "utils/rel.h" #include "utils/snapmgr.h" -static bool toastrel_valueid_exists(Relation toastrel, Oid valueid); -static bool toastid_valueid_exists(Oid toastrelid, Oid valueid); +static bool toastrel_valueid_exists(Relation toastrel, uint64 valueid); +static bool toastid_valueid_exists(Oid toastrelid, uint64 valueid); /* ---------- * toast_compress_datum - @@ -456,7 +456,7 @@ toast_delete_datum(Relation rel, Datum value, bool is_speculative) * ---------- */ static bool -toastrel_valueid_exists(Relation toastrel, Oid valueid) +toastrel_valueid_exists(Relation toastrel, uint64 valueid) { bool result = false; ScanKeyData toastkey; @@ -504,7 +504,7 @@ toastrel_valueid_exists(Relation toastrel, Oid valueid) * ---------- */ static bool -toastid_valueid_exists(Oid toastrelid, Oid valueid) +toastid_valueid_exists(Oid toastrelid, uint64 valueid) { bool result; Relation toastrel; diff --git a/src/backend/access/heap/heaptoast.c b/src/backend/access/heap/heaptoast.c index cb1e57030f64..76936b2f4944 100644 --- a/src/backend/access/heap/heaptoast.c +++ b/src/backend/access/heap/heaptoast.c @@ -623,7 +623,7 @@ toast_build_flattened_tuple(TupleDesc tupleDesc, * result is the varlena into which the results should be written. */ void -heap_fetch_toast_slice(Relation toastrel, Oid valueid, int32 attrsize, +heap_fetch_toast_slice(Relation toastrel, uint64 valueid, int32 attrsize, int32 sliceoffset, int32 slicelength, struct varlena *result) { @@ -725,7 +725,7 @@ heap_fetch_toast_slice(Relation toastrel, Oid valueid, int32 attrsize, else { /* should never happen */ - elog(ERROR, "found toasted toast chunk for toast value %u in %s", + elog(ERROR, "found toasted toast chunk for toast value %" PRIu64 " in %s", valueid, RelationGetRelationName(toastrel)); chunksize = 0; /* keep compiler quiet */ chunkdata = NULL; @@ -737,13 +737,13 @@ heap_fetch_toast_slice(Relation toastrel, Oid valueid, int32 attrsize, if (curchunk != expectedchunk) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), - errmsg_internal("unexpected chunk number %d (expected %d) for toast value %u in %s", + errmsg_internal("unexpected chunk number %d (expected %d) for toast value %" PRIu64 " in %s", curchunk, expectedchunk, valueid, RelationGetRelationName(toastrel)))); if (curchunk > endchunk) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), - errmsg_internal("unexpected chunk number %d (out of range %d..%d) for toast value %u in %s", + errmsg_internal("unexpected chunk number %d (out of range %d..%d) for toast value %" PRIu64 " in %s", curchunk, startchunk, endchunk, valueid, RelationGetRelationName(toastrel)))); @@ -752,7 +752,7 @@ heap_fetch_toast_slice(Relation toastrel, Oid valueid, int32 attrsize, if (chunksize != expected_size) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), - errmsg_internal("unexpected chunk size %d (expected %d) in chunk %d of %d for toast value %u in %s", + errmsg_internal("unexpected chunk size %d (expected %d) in chunk %d of %d for toast value %" PRIu64 " in %s", chunksize, expected_size, curchunk, totalchunks, valueid, RelationGetRelationName(toastrel)))); @@ -781,7 +781,7 @@ heap_fetch_toast_slice(Relation toastrel, Oid valueid, int32 attrsize, if (expectedchunk != (endchunk + 1)) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), - errmsg_internal("missing chunk number %d for toast value %u in %s", + errmsg_internal("missing chunk number %d for toast value %" PRIu64 " in %s", expectedchunk, valueid, RelationGetRelationName(toastrel)))); diff --git a/src/backend/replication/logical/reorderbuffer.c b/src/backend/replication/logical/reorderbuffer.c index 7b4e8629553b..c871708b5932 100644 --- a/src/backend/replication/logical/reorderbuffer.c +++ b/src/backend/replication/logical/reorderbuffer.c @@ -176,7 +176,7 @@ typedef struct ReorderBufferIterTXNState /* toast datastructures */ typedef struct ReorderBufferToastEnt { - Oid chunk_id; /* toast_table.chunk_id */ + uint64 chunk_id; /* toast_table.chunk_id */ int32 last_chunk_seq; /* toast_table.chunk_seq of the last chunk we * have seen */ Size num_chunks; /* number of chunks we've already seen */ @@ -4944,7 +4944,7 @@ ReorderBufferToastInitHash(ReorderBuffer *rb, ReorderBufferTXN *txn) Assert(txn->toast_hash == NULL); - hash_ctl.keysize = sizeof(Oid); + hash_ctl.keysize = sizeof(uint64); hash_ctl.entrysize = sizeof(ReorderBufferToastEnt); hash_ctl.hcxt = rb->context; txn->toast_hash = hash_create("ReorderBufferToastHash", 5, &hash_ctl, @@ -4968,7 +4968,7 @@ ReorderBufferToastAppendChunk(ReorderBuffer *rb, ReorderBufferTXN *txn, bool isnull; Pointer chunk; TupleDesc desc = RelationGetDescr(relation); - Oid chunk_id; + uint64 chunk_id; int32 chunk_seq; if (txn->toast_hash == NULL) @@ -4995,11 +4995,11 @@ ReorderBufferToastAppendChunk(ReorderBuffer *rb, ReorderBufferTXN *txn, dlist_init(&ent->chunks); if (chunk_seq != 0) - elog(ERROR, "got sequence entry %d for toast chunk %u instead of seq 0", + elog(ERROR, "got sequence entry %d for toast chunk %" PRIu64 " instead of seq 0", chunk_seq, chunk_id); } else if (found && chunk_seq != ent->last_chunk_seq + 1) - elog(ERROR, "got sequence entry %d for toast chunk %u instead of seq %d", + elog(ERROR, "got sequence entry %d for toast chunk %" PRIu64 " instead of seq %d", chunk_seq, chunk_id, ent->last_chunk_seq + 1); chunk = DatumGetPointer(fastgetattr(newtup, 3, desc, &isnull)); @@ -5108,6 +5108,7 @@ ReorderBufferToastReplace(ReorderBuffer *rb, ReorderBufferTXN *txn, struct varlena *reconstructed; dlist_iter it; Size data_done = 0; + uint64 toast_valueid; /* system columns aren't toasted */ if (attr->attnum < 0) @@ -5132,13 +5133,14 @@ ReorderBufferToastReplace(ReorderBuffer *rb, ReorderBufferTXN *txn, continue; VARATT_EXTERNAL_GET_POINTER(toast_pointer, varlena); + toast_valueid = toast_pointer.va_valueid; /* * Check whether the toast tuple changed, replace if so. */ ent = (ReorderBufferToastEnt *) hash_search(txn->toast_hash, - &toast_pointer.va_valueid, + &toast_valueid, HASH_FIND, NULL); if (ent == NULL) diff --git a/contrib/amcheck/verify_heapam.c b/contrib/amcheck/verify_heapam.c index 4963e9245cb5..3b2bdced4cdc 100644 --- a/contrib/amcheck/verify_heapam.c +++ b/contrib/amcheck/verify_heapam.c @@ -1556,11 +1556,18 @@ check_toast_tuple(HeapTuple toasttup, HeapCheckContext *ctx, uint32 extsize) { int32 chunk_seq; - int32 last_chunk_seq = (extsize - 1) / TOAST_MAX_CHUNK_SIZE; + int32 last_chunk_seq; Pointer chunk; bool isnull; int32 chunksize; int32 expected_size; + uint64 toast_valueid; + int32 max_chunk_size; + + toast_valueid = ta->toast_pointer.va_valueid; + + max_chunk_size = TOAST_MAX_CHUNK_SIZE; + last_chunk_seq = (extsize - 1) / max_chunk_size; /* Sanity-check the sequence number. */ chunk_seq = DatumGetInt32(fastgetattr(toasttup, 2, @@ -1568,16 +1575,16 @@ check_toast_tuple(HeapTuple toasttup, HeapCheckContext *ctx, if (isnull) { report_toast_corruption(ctx, ta, - psprintf("toast value %u has toast chunk with null sequence number", - ta->toast_pointer.va_valueid)); + psprintf("toast value %" PRIu64 " has toast chunk with null sequence number", + toast_valueid)); return; } if (chunk_seq != *expected_chunk_seq) { /* Either the TOAST index is corrupt, or we don't have all chunks. */ report_toast_corruption(ctx, ta, - psprintf("toast value %u index scan returned chunk %d when expecting chunk %d", - ta->toast_pointer.va_valueid, + psprintf("toast value %" PRIu64 " index scan returned chunk %d when expecting chunk %d", + toast_valueid, chunk_seq, *expected_chunk_seq)); } *expected_chunk_seq = chunk_seq + 1; @@ -1588,8 +1595,8 @@ check_toast_tuple(HeapTuple toasttup, HeapCheckContext *ctx, if (isnull) { report_toast_corruption(ctx, ta, - psprintf("toast value %u chunk %d has null data", - ta->toast_pointer.va_valueid, + psprintf("toast value %" PRIu64 " chunk %d has null data", + toast_valueid, chunk_seq)); return; } @@ -1608,8 +1615,8 @@ check_toast_tuple(HeapTuple toasttup, HeapCheckContext *ctx, uint32 header = ((varattrib_4b *) chunk)->va_4byte.va_header; report_toast_corruption(ctx, ta, - psprintf("toast value %u chunk %d has invalid varlena header %0x", - ta->toast_pointer.va_valueid, + psprintf("toast value %" PRIu64 " chunk %d has invalid varlena header %0x", + toast_valueid, chunk_seq, header)); return; } @@ -1620,19 +1627,19 @@ check_toast_tuple(HeapTuple toasttup, HeapCheckContext *ctx, if (chunk_seq > last_chunk_seq) { report_toast_corruption(ctx, ta, - psprintf("toast value %u chunk %d follows last expected chunk %d", - ta->toast_pointer.va_valueid, + psprintf("toast value %" PRIu64 " chunk %d follows last expected chunk %d", + toast_valueid, chunk_seq, last_chunk_seq)); return; } - expected_size = chunk_seq < last_chunk_seq ? TOAST_MAX_CHUNK_SIZE - : extsize - (last_chunk_seq * TOAST_MAX_CHUNK_SIZE); + expected_size = chunk_seq < last_chunk_seq ? max_chunk_size + : extsize - (last_chunk_seq * max_chunk_size); if (chunksize != expected_size) report_toast_corruption(ctx, ta, - psprintf("toast value %u chunk %d has size %u, but expected size %u", - ta->toast_pointer.va_valueid, + psprintf("toast value %" PRIu64 " chunk %d has size %u, but expected size %u", + toast_valueid, chunk_seq, chunksize, expected_size)); } @@ -1663,6 +1670,7 @@ check_tuple_attribute(HeapCheckContext *ctx) struct varlena *attr; char *tp; /* pointer to the tuple data */ uint16 infomask; + uint64 toast_pointer_valueid; CompactAttribute *thisatt; struct varatt_external toast_pointer; @@ -1766,6 +1774,7 @@ check_tuple_attribute(HeapCheckContext *ctx) return true; /* It is external, and we're looking at a page on disk */ + toast_pointer_valueid = toast_pointer.va_valueid; /* * Must copy attr into toast_pointer for alignment considerations @@ -1775,8 +1784,8 @@ check_tuple_attribute(HeapCheckContext *ctx) /* Toasted attributes too large to be untoasted should never be stored */ if (toast_pointer.va_rawsize > VARLENA_SIZE_LIMIT) report_corruption(ctx, - psprintf("toast value %u rawsize %d exceeds limit %d", - toast_pointer.va_valueid, + psprintf("toast value %" PRIu64 " rawsize %d exceeds limit %d", + toast_pointer_valueid, toast_pointer.va_rawsize, VARLENA_SIZE_LIMIT)); @@ -1803,16 +1812,16 @@ check_tuple_attribute(HeapCheckContext *ctx) } if (!valid) report_corruption(ctx, - psprintf("toast value %u has invalid compression method id %d", - toast_pointer.va_valueid, cmid)); + psprintf("toast value %" PRIu64 " has invalid compression method id %d", + toast_pointer_valueid, cmid)); } /* The tuple header better claim to contain toasted values */ if (!(infomask & HEAP_HASEXTERNAL)) { report_corruption(ctx, - psprintf("toast value %u is external but tuple header flag HEAP_HASEXTERNAL not set", - toast_pointer.va_valueid)); + psprintf("toast value %" PRIu64 " is external but tuple header flag HEAP_HASEXTERNAL not set", + toast_pointer_valueid)); return true; } @@ -1820,8 +1829,8 @@ check_tuple_attribute(HeapCheckContext *ctx) if (!ctx->rel->rd_rel->reltoastrelid) { report_corruption(ctx, - psprintf("toast value %u is external but relation has no toast relation", - toast_pointer.va_valueid)); + psprintf("toast value %" PRIu64 " is external but relation has no toast relation", + toast_pointer_valueid)); return true; } @@ -1866,9 +1875,11 @@ check_toasted_attribute(HeapCheckContext *ctx, ToastedAttribute *ta) uint32 extsize; int32 expected_chunk_seq = 0; int32 last_chunk_seq; + uint64 toast_valueid; + int32 max_chunk_size = TOAST_MAX_CHUNK_SIZE; extsize = VARATT_EXTERNAL_GET_EXTSIZE(ta->toast_pointer); - last_chunk_seq = (extsize - 1) / TOAST_MAX_CHUNK_SIZE; + last_chunk_seq = (extsize - 1) / max_chunk_size; /* * Setup a scan key to find chunks in toast table with matching va_valueid @@ -1896,14 +1907,16 @@ check_toasted_attribute(HeapCheckContext *ctx, ToastedAttribute *ta) } systable_endscan_ordered(toastscan); + toast_valueid = ta->toast_pointer.va_valueid; + if (!found_toasttup) report_toast_corruption(ctx, ta, - psprintf("toast value %u not found in toast table", - ta->toast_pointer.va_valueid)); + psprintf("toast value %" PRIu64 " not found in toast table", + toast_valueid)); else if (expected_chunk_seq <= last_chunk_seq) report_toast_corruption(ctx, ta, - psprintf("toast value %u was expected to end at chunk %d, but ended while expecting chunk %d", - ta->toast_pointer.va_valueid, + psprintf("toast value %" PRIu64 " was expected to end at chunk %d, but ended while expecting chunk %d", + toast_valueid, last_chunk_seq, expected_chunk_seq)); } -- 2.50.0