From a53b3f59a7d01f01d21d7f19fd6d34a9a3860228 Mon Sep 17 00:00:00 2001 From: Ayush Tiwari Date: Wed, 17 Jun 2026 22:52:36 +0530 Subject: [PATCH v1] Fix btree_gist <> strategy on internal index pages gbt_var_consistent() handled the <> (BtreeGistNotEqual) strategy without distinguishing leaf from internal pages, unlike every other strategy. Internal keys are lossy, truncated prefixes, and for bit/varbit the leaf-to-node transform drops the VarBit length header, so such a key is not a valid value of the indexed type. Passing it to the type's equality function is a type confusion. For bit/varbit it reaches bit_cmp(), where VARBITBYTES() computes VARSIZE - 8; a key with VARSIZE < 8 yields a negative length that, passed to memcmp() as size_t, becomes huge and reads out of bounds (eventually crashing the backend). Even without a crash it can give wrong equality results and defeat an exclusion constraint declared with <>. Make the <> branch distinguish leaf and internal pages: on a leaf, negate the equality test against the stored value; on an internal page, recurse unconditionally. The fix is in the shared gbt_var_consistent() and so covers every variable-length type. Add a regression test. --- contrib/btree_gist/btree_utils_var.c | 14 +++++++++++-- contrib/btree_gist/expected/not_equal.out | 24 +++++++++++++++++++++++ contrib/btree_gist/sql/not_equal.sql | 18 +++++++++++++++++ 3 files changed, 54 insertions(+), 2 deletions(-) diff --git a/contrib/btree_gist/btree_utils_var.c b/contrib/btree_gist/btree_utils_var.c index 25c3bbe8eac..6043dd811bc 100644 --- a/contrib/btree_gist/btree_utils_var.c +++ b/contrib/btree_gist/btree_utils_var.c @@ -611,8 +611,18 @@ gbt_var_consistent(GBT_VARKEY_R *key, || gbt_var_node_pf_match(key, query, tinfo); break; case BtreeGistNotEqualStrategyNumber: - retval = !(tinfo->f_eq(query, key->lower, collation, flinfo) && - tinfo->f_eq(query, key->upper, collation, flinfo)); + + /* + * Must distinguish leaf and internal pages. Internal keys are + * lossy, truncated prefixes whose representation may differ from + * the leaf form (e.g. bit/varbit), so f_eq() on them can give + * wrong answers or read out of bounds. An internal page may + * always hold a non-equal value, so just recurse into it. + */ + if (is_leaf) + retval = !tinfo->f_eq(query, key->lower, collation, flinfo); + else + retval = true; break; default: retval = false; diff --git a/contrib/btree_gist/expected/not_equal.out b/contrib/btree_gist/expected/not_equal.out index 85b1e868a87..922950e0c2f 100644 --- a/contrib/btree_gist/expected/not_equal.out +++ b/contrib/btree_gist/expected/not_equal.out @@ -39,3 +39,27 @@ INSERT INTO zoo VALUES(123, 'lion'); ERROR: conflicting key value violates exclusion constraint "zoo_cage_animal_excl" DETAIL: Key (cage, animal)=(123, lion) conflicts with existing key (cage, animal)=(123, zebra). INSERT INTO zoo VALUES(124, 'lion'); +-- A <> search must descend internal pages instead of comparing the query to a +-- truncated internal key. Use a multi-level index and check it agrees with seq scan. +CREATE TABLE bitne (a bit(8)); +INSERT INTO bitne SELECT (g % 256)::bit(8) FROM generate_series(1, 20000) g; +CREATE INDEX bitne_idx ON bitne USING gist (a); +SET enable_seqscan to on; +SET enable_indexscan to off; +SET enable_bitmapscan to off; +SELECT count(*) FROM bitne WHERE a <> B'00000000'; + count +------- + 19922 +(1 row) + +SET enable_seqscan to off; +SET enable_indexscan to on; +SET enable_bitmapscan to on; +SELECT count(*) FROM bitne WHERE a <> B'00000000'; + count +------- + 19922 +(1 row) + +DROP TABLE bitne; diff --git a/contrib/btree_gist/sql/not_equal.sql b/contrib/btree_gist/sql/not_equal.sql index 6dfac5d0aae..ecb135f47d2 100644 --- a/contrib/btree_gist/sql/not_equal.sql +++ b/contrib/btree_gist/sql/not_equal.sql @@ -34,3 +34,21 @@ INSERT INTO zoo VALUES(123, 'zebra'); INSERT INTO zoo VALUES(123, 'zebra'); INSERT INTO zoo VALUES(123, 'lion'); INSERT INTO zoo VALUES(124, 'lion'); + +-- A <> search must descend internal pages instead of comparing the query to a +-- truncated internal key. Use a multi-level index and check it agrees with seq scan. +CREATE TABLE bitne (a bit(8)); +INSERT INTO bitne SELECT (g % 256)::bit(8) FROM generate_series(1, 20000) g; +CREATE INDEX bitne_idx ON bitne USING gist (a); + +SET enable_seqscan to on; +SET enable_indexscan to off; +SET enable_bitmapscan to off; +SELECT count(*) FROM bitne WHERE a <> B'00000000'; + +SET enable_seqscan to off; +SET enable_indexscan to on; +SET enable_bitmapscan to on; +SELECT count(*) FROM bitne WHERE a <> B'00000000'; + +DROP TABLE bitne; -- 2.34.1