From c82d6c992966d0e6e9da6d974691b1e91a2310ac Mon Sep 17 00:00:00 2001 From: "Andrei V. Lepikhov" Date: Mon, 10 Aug 2026 20:17:03 +0200 Subject: [PATCH v0 1/3] Unpack short numerics on the local buffer The hash, comparison and sortsupport functions for numeric are hot, but every call ran DatumGetNumeric() on its input. Numerics come out of a tuple with a one-byte varlena header and need one allocation per value per call. None of these functions lets the value outlive the call, so unpack into a local buffer, falling back to DatumGetNumeric() only for compressed and out-of-line values. The buffer is sized from the varlena short-header, so that every value which can arrive with a one-byte header fits in it. The comparison functions free a detoasted copy, as they did before. The hash functions never freed theirs; they keep leaving it to the caller's memory context, so for them PG_GETARG_NUMERIC_LOCAL() simply replaces PG_GETARG_NUMERIC(). --- src/backend/utils/adt/numeric.c | 189 +++++++++++++++++-------------- src/tools/pgindent/typedefs.list | 1 + 2 files changed, 104 insertions(+), 86 deletions(-) diff --git a/src/backend/utils/adt/numeric.c b/src/backend/utils/adt/numeric.c index 7deea2046f1..965ff8c27e5 100644 --- a/src/backend/utils/adt/numeric.c +++ b/src/backend/utils/adt/numeric.c @@ -494,6 +494,65 @@ static void dump_var(const char *str, NumericVar *var); (weight) <= NUMERIC_SHORT_WEIGHT_MAX && \ (weight) >= NUMERIC_SHORT_WEIGHT_MIN) +#define NUMERIC_LOCAL_DATASZ (VARATT_SHORT_MAX - VARHDRSZ_SHORT) + +typedef union NumericLocalBuf +{ + struct NumericData num; + char data[VARHDRSZ + NUMERIC_LOCAL_DATASZ]; +} NumericLocalBuf; + +/* + * Return a four-byte-header Numeric for X, using *buf when possible. Sets + * *needfree if the result is a palloc'd detoasted copy. Callers that must + * not leak memory, such as the comparison functions, pfree it. The others use + * DatumGetNumericLocal() below and leave the copy to the caller's memory + * context. + */ +static inline Numeric +numeric_unpack_local(Datum X, NumericLocalBuf *buf, bool *needfree) +{ + varlena *attr = (varlena *) DatumGetPointer(X); + + *needfree = false; + + /* + * VARATT_IS_SHORT() is true for external TOAST pointers too, for which + * VARSIZE_SHORT() returns 0. Exclude them here so that they reach the + * detoasting path below instead of the copy. + */ + if (VARATT_IS_SHORT(attr) && !VARATT_IS_EXTERNAL(attr)) + { + Size data_size = VARSIZE_SHORT(attr) - VARHDRSZ_SHORT; + + SET_VARSIZE(&buf->num, data_size + VARHDRSZ); + memcpy(VARDATA(&buf->num), VARDATA_SHORT(attr), data_size); + return &buf->num; + } + else if (likely(!VARATT_IS_EXTENDED(attr))) + return (Numeric) attr; + + /* compressed or out-of-line */ + *needfree = true; + return DatumGetNumeric(X); +} + +/* + * DatumGetNumeric() that unpacks a short-header value into *buf instead of + * palloc'ing a copy. The result may point into *buf, so it must not be + * returned or kept beyond the caller's frame. + */ +static inline Numeric +DatumGetNumericLocal(Datum X, NumericLocalBuf *buf) +{ + bool needfree; + + return numeric_unpack_local(X, buf, &needfree); +} + +#define PG_GETARG_NUMERIC_LOCAL(n, buf) \ + DatumGetNumericLocal(PG_GETARG_DATUM(n), (buf)) + static void alloc_var(NumericVar *var, int ndigits); static void free_var(NumericVar *var); static void zero_var(NumericVar *var); @@ -2273,32 +2332,44 @@ numeric_abbrev_abort(int memtupcount, SortSupport ssup) } /* - * Non-fmgr interface to the comparison routine to allow sortsupport to elide - * the fmgr call. The saving here is small given how slow numeric comparisons - * are, but it is a required part of the sort support API when abbreviations - * are performed. - * - * Two palloc/pfree cycles could be saved here by using persistent buffers for - * aligning short-varlena inputs, but this has not so far been considered to - * be worth the effort. + * Compare two numeric Datums without palloc'ing short-header inputs; see + * numeric_unpack_local(). This is the body of numeric_fast_cmp() and of all + * the fmgr-callable comparison functions below; it is forced inline so that + * each of them stays a single call into cmp_numerics(). */ -static int -numeric_fast_cmp(Datum x, Datum y, SortSupport ssup) +static pg_always_inline int +cmp_numeric_datums(Datum x, Datum y) { - Numeric nx = DatumGetNumeric(x); - Numeric ny = DatumGetNumeric(y); + NumericLocalBuf bufx; + NumericLocalBuf bufy; + bool freex; + bool freey; + Numeric nx = numeric_unpack_local(x, &bufx, &freex); + Numeric ny = numeric_unpack_local(y, &bufy, &freey); int result; result = cmp_numerics(nx, ny); - if (nx != DatumGetPointer(x)) + if (unlikely(freex)) pfree(nx); - if (ny != DatumGetPointer(y)) + if (unlikely(freey)) pfree(ny); return result; } +/* + * Non-fmgr interface to the comparison routine to allow sortsupport to elide + * the fmgr call. The saving here is small given how slow numeric comparisons + * are, but it is a required part of the sort support API when abbreviations + * are performed. + */ +static int +numeric_fast_cmp(Datum x, Datum y, SortSupport ssup) +{ + return cmp_numeric_datums(x, y); +} + /* * Compare abbreviations of values. (Abbreviations may be equal where the true * values differ, but if the abbreviations differ, they must reflect the @@ -2418,107 +2489,51 @@ numeric_abbrev_convert_var(const NumericVar *var, NumericSortSupport *nss) Datum numeric_cmp(PG_FUNCTION_ARGS) { - Numeric num1 = PG_GETARG_NUMERIC(0); - Numeric num2 = PG_GETARG_NUMERIC(1); - int result; - - result = cmp_numerics(num1, num2); - - PG_FREE_IF_COPY(num1, 0); - PG_FREE_IF_COPY(num2, 1); - - PG_RETURN_INT32(result); + PG_RETURN_INT32(cmp_numeric_datums(PG_GETARG_DATUM(0), + PG_GETARG_DATUM(1))); } Datum numeric_eq(PG_FUNCTION_ARGS) { - Numeric num1 = PG_GETARG_NUMERIC(0); - Numeric num2 = PG_GETARG_NUMERIC(1); - bool result; - - result = cmp_numerics(num1, num2) == 0; - - PG_FREE_IF_COPY(num1, 0); - PG_FREE_IF_COPY(num2, 1); - - PG_RETURN_BOOL(result); + PG_RETURN_BOOL(cmp_numeric_datums(PG_GETARG_DATUM(0), + PG_GETARG_DATUM(1)) == 0); } Datum numeric_ne(PG_FUNCTION_ARGS) { - Numeric num1 = PG_GETARG_NUMERIC(0); - Numeric num2 = PG_GETARG_NUMERIC(1); - bool result; - - result = cmp_numerics(num1, num2) != 0; - - PG_FREE_IF_COPY(num1, 0); - PG_FREE_IF_COPY(num2, 1); - - PG_RETURN_BOOL(result); + PG_RETURN_BOOL(cmp_numeric_datums(PG_GETARG_DATUM(0), + PG_GETARG_DATUM(1)) != 0); } Datum numeric_gt(PG_FUNCTION_ARGS) { - Numeric num1 = PG_GETARG_NUMERIC(0); - Numeric num2 = PG_GETARG_NUMERIC(1); - bool result; - - result = cmp_numerics(num1, num2) > 0; - - PG_FREE_IF_COPY(num1, 0); - PG_FREE_IF_COPY(num2, 1); - - PG_RETURN_BOOL(result); + PG_RETURN_BOOL(cmp_numeric_datums(PG_GETARG_DATUM(0), + PG_GETARG_DATUM(1)) > 0); } Datum numeric_ge(PG_FUNCTION_ARGS) { - Numeric num1 = PG_GETARG_NUMERIC(0); - Numeric num2 = PG_GETARG_NUMERIC(1); - bool result; - - result = cmp_numerics(num1, num2) >= 0; - - PG_FREE_IF_COPY(num1, 0); - PG_FREE_IF_COPY(num2, 1); - - PG_RETURN_BOOL(result); + PG_RETURN_BOOL(cmp_numeric_datums(PG_GETARG_DATUM(0), + PG_GETARG_DATUM(1)) >= 0); } Datum numeric_lt(PG_FUNCTION_ARGS) { - Numeric num1 = PG_GETARG_NUMERIC(0); - Numeric num2 = PG_GETARG_NUMERIC(1); - bool result; - - result = cmp_numerics(num1, num2) < 0; - - PG_FREE_IF_COPY(num1, 0); - PG_FREE_IF_COPY(num2, 1); - - PG_RETURN_BOOL(result); + PG_RETURN_BOOL(cmp_numeric_datums(PG_GETARG_DATUM(0), + PG_GETARG_DATUM(1)) < 0); } Datum numeric_le(PG_FUNCTION_ARGS) { - Numeric num1 = PG_GETARG_NUMERIC(0); - Numeric num2 = PG_GETARG_NUMERIC(1); - bool result; - - result = cmp_numerics(num1, num2) <= 0; - - PG_FREE_IF_COPY(num1, 0); - PG_FREE_IF_COPY(num2, 1); - - PG_RETURN_BOOL(result); + PG_RETURN_BOOL(cmp_numeric_datums(PG_GETARG_DATUM(0), + PG_GETARG_DATUM(1)) <= 0); } static int @@ -2716,7 +2731,8 @@ in_range_numeric_numeric(PG_FUNCTION_ARGS) Datum hash_numeric(PG_FUNCTION_ARGS) { - Numeric key = PG_GETARG_NUMERIC(0); + NumericLocalBuf buf; + Numeric key = PG_GETARG_NUMERIC_LOCAL(0, &buf); uint32 digit_hash; uint32 result; int weight; @@ -2797,7 +2813,8 @@ hash_numeric(PG_FUNCTION_ARGS) Datum hash_numeric_extended(PG_FUNCTION_ARGS) { - Numeric key = PG_GETARG_NUMERIC(0); + NumericLocalBuf buf; + Numeric key = PG_GETARG_NUMERIC_LOCAL(0, &buf); uint64 seed = PG_GETARG_INT64(1); uint64 digit_hash; uint64 result; diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f10eb469266..6445598e167 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -1845,6 +1845,7 @@ NullingRelsMatch Numeric NumericAggState NumericDigit +NumericLocalBuf NumericSortSupport NumericSumAccum NumericVar -- 2.56.1