From 5af86ee09cc3a28b22067ad543919f0cbcc31157 Mon Sep 17 00:00:00 2001 From: Ayush Tiwari Date: Sat, 13 Jun 2026 11:35:20 +0530 Subject: [PATCH v1] Fix int32 overflow in ltree_compare() The expression (len_diff * 10 * (an + 1)) used as the return value of ltree_compare() is computed at int32 width. With LTREE_MAX_LEVELS = 65535, the product can exceed INT32_MAX once an ltree has more than ~14,653 levels, which causes the result to wrap and invert its sign. That corrupts btree ordering as well as the magnitude consumed by ltree_penalty() for GiST page splits. Widen the multiplication to int64 and saturate the result to PG_INT32_MIN / PG_INT32_MAX before returning. Saturation preserves both the sign needed by btree comparators and the "deeper mismatch yields larger magnitude" property used by ltree_penalty(). Returning INT_MIN from a comparator has been explicitly allowed since commit 6e63e069751. ltree_penalty() sums the two comparator results, so a pair of saturated INT32_MAX values would re-overflow int32 and yield a negative penalty once assigned to its float output. Widen that sum to int64 before converting to float. Existing btree or GiST indexes on ltree columns containing values with more than ~14,653 levels may be corrupt and should be REINDEXed. Add a regression test based on the reporter's PoC. --- contrib/ltree/expected/ltree.out | 13 +++++++++++++ contrib/ltree/ltree_gist.c | 2 +- contrib/ltree/ltree_op.c | 29 ++++++++++++++++++++++++++--- contrib/ltree/sql/ltree.sql | 9 +++++++++ 4 files changed, 49 insertions(+), 4 deletions(-) diff --git a/contrib/ltree/expected/ltree.out b/contrib/ltree/expected/ltree.out index 15b9131a750..9d83aa2fa84 100644 --- a/contrib/ltree/expected/ltree.out +++ b/contrib/ltree/expected/ltree.out @@ -8226,3 +8226,16 @@ DETAIL: Total size of level exceeds the maximum allowed (65535 bytes). SELECT (repeat('a|', 65535) || 'a')::lquery; ERROR: lquery level has too many variants DETAIL: Number of variants exceeds the maximum allowed (65535). +-- Test that ltree_compare() does not overflow int32 with very deep paths. +-- Without saturation, the product (len_diff * 10 * (an + 1)) wraps past +-- INT32_MAX above ~14653 levels and the sign of the result is inverted, +-- which corrupts btree ordering and GiST page splits. +WITH s AS (SELECT 'a'::ltree AS v), + l AS (SELECT (repeat('a.', 14999) || 'a')::ltree AS v) +SELECT (l.v > s.v) AS gt_ok, (l.v < s.v) AS lt_ok, (l.v = s.v) AS eq_ok + FROM s, l; + gt_ok | lt_ok | eq_ok +-------+-------+------- + t | f | f +(1 row) + diff --git a/contrib/ltree/ltree_gist.c b/contrib/ltree/ltree_gist.c index 78c95052990..433597de2da 100644 --- a/contrib/ltree/ltree_gist.c +++ b/contrib/ltree/ltree_gist.c @@ -270,7 +270,7 @@ ltree_penalty(PG_FUNCTION_ARGS) cmpl = ltree_compare(LTG_GETLNODE(origval, siglen), LTG_GETLNODE(newval, siglen)); cmpr = ltree_compare(LTG_GETRNODE(newval, siglen), LTG_GETRNODE(origval, siglen)); - *penalty = Max(cmpl, 0) + Max(cmpr, 0); + *penalty = (float) ((int64) Max(cmpl, 0) + (int64) Max(cmpr, 0)); PG_RETURN_POINTER(penalty); } diff --git a/contrib/ltree/ltree_op.c b/contrib/ltree/ltree_op.c index c1fc77fc804..f2f5ad9fd0f 100644 --- a/contrib/ltree/ltree_op.c +++ b/contrib/ltree/ltree_op.c @@ -57,15 +57,30 @@ ltree_compare(const ltree *a, const ltree *b) if ((res = memcmp(al->name, bl->name, Min(al->len, bl->len))) == 0) { if (al->len != bl->len) - return (al->len - bl->len) * 10 * (an + 1); + { + int64 v = (int64) (al->len - bl->len) * 10 * (an + 1); + + if (v > PG_INT32_MAX) + return PG_INT32_MAX; + if (v < PG_INT32_MIN) + return PG_INT32_MIN; + return (int32) v; + } } else { + int64 v; + if (res < 0) res = -1; else res = 1; - return res * 10 * (an + 1); + v = (int64) res * 10 * (an + 1); + if (v > PG_INT32_MAX) + return PG_INT32_MAX; + if (v < PG_INT32_MIN) + return PG_INT32_MIN; + return (int32) v; } an--; @@ -74,7 +89,15 @@ ltree_compare(const ltree *a, const ltree *b) bl = LEVEL_NEXT(bl); } - return (a->numlevel - b->numlevel) * 10 * (an + 1); + { + int64 v = (int64) (a->numlevel - b->numlevel) * 10 * (an + 1); + + if (v > PG_INT32_MAX) + return PG_INT32_MAX; + if (v < PG_INT32_MIN) + return PG_INT32_MIN; + return (int32) v; + } } #define RUNCMP \ diff --git a/contrib/ltree/sql/ltree.sql b/contrib/ltree/sql/ltree.sql index d0fade9d17d..3e9c1ff79ee 100644 --- a/contrib/ltree/sql/ltree.sql +++ b/contrib/ltree/sql/ltree.sql @@ -477,3 +477,12 @@ SELECT (repeat('x', 255) || repeat('|' || repeat('x', 255), 256))::lquery; --- Test for overflow of lquery_level.numvar, with a set of single-char --- variants in one level. SELECT (repeat('a|', 65535) || 'a')::lquery; + +-- Test that ltree_compare() does not overflow int32 with very deep paths. +-- Without saturation, the product (len_diff * 10 * (an + 1)) wraps past +-- INT32_MAX above ~14653 levels and the sign of the result is inverted, +-- which corrupts btree ordering and GiST page splits. +WITH s AS (SELECT 'a'::ltree AS v), + l AS (SELECT (repeat('a.', 14999) || 'a')::ltree AS v) +SELECT (l.v > s.v) AS gt_ok, (l.v < s.v) AS lt_ok, (l.v = s.v) AS eq_ok + FROM s, l; -- 2.34.1