From 47571c5dca77142ed3ace8ed121aad89806c3a04 Mon Sep 17 00:00:00 2001
From: Zsolt Parragi <zsolt.parragi@percona.com>
Date: Wed, 12 Aug 2026 14:53:10 +0000
Subject: [PATCH 2/2] Accept maximum-length lexemes in all text search input
 paths

MAXSTRLEN is the maximum value of WordEntry.len, so a lexeme of exactly
that length (2047 bytes) fits in the tsvector on-disk format, and both
tsvectorrecv() and tsqueryrecv() accept one.  tsvectorin(), tsqueryin()
and the token length check in the text search parser rejected it,
though, reporting a maximum that is one byte short of the real one.

The inconsistency means a tsvector received in binary format, or built
by array_to_tsvector(), can hold a lexeme that its own text output
cannot be read back in: a plain pg_dump of such a value fails to
restore with "word is too long (2047 bytes, max 2046 bytes)".  The
parser is inconsistent with itself as well, as the length check applied
to dictionary output already allows MAXSTRLEN bytes.

Fix by rejecting only lexemes longer than MAXSTRLEN everywhere.  This
also matches the documented limit of less than 2 kilobytes per lexeme.
---
 src/backend/tsearch/ts_parse.c        |  4 ++--
 src/backend/utils/adt/tsquery.c       |  4 ++--
 src/backend/utils/adt/tsvector.c      |  4 ++--
 src/test/regress/expected/tsearch.out | 15 ++++++++++++++
 src/test/regress/expected/tstypes.out | 28 +++++++++++++++++++++++++++
 src/test/regress/sql/tsearch.sql      |  4 ++++
 src/test/regress/sql/tstypes.sql      |  7 +++++++
 7 files changed, 60 insertions(+), 6 deletions(-)

diff --git a/src/backend/tsearch/ts_parse.c b/src/backend/tsearch/ts_parse.c
index cb69e9899c5..93d46f1d642 100644
--- a/src/backend/tsearch/ts_parse.c
+++ b/src/backend/tsearch/ts_parse.c
@@ -379,7 +379,7 @@ parsetext(Oid cfgId, ParsedText *prs, char *buf, int buflen)
 										   PointerGetDatum(&lemm),
 										   PointerGetDatum(&lenlemm)));
 
-		if (type > 0 && lenlemm >= MAXSTRLEN)
+		if (type > 0 && lenlemm > MAXSTRLEN)
 		{
 #ifdef IGNORE_LONGLEXEME
 			ereport(NOTICE,
@@ -582,7 +582,7 @@ hlparsetext(Oid cfgId, HeadlineParsedText *prs, TSQuery query, char *buf, int bu
 										   PointerGetDatum(&lemm),
 										   PointerGetDatum(&lenlemm)));
 
-		if (type > 0 && lenlemm >= MAXSTRLEN)
+		if (type > 0 && lenlemm > MAXSTRLEN)
 		{
 #ifdef IGNORE_LONGLEXEME
 			ereport(NOTICE,
diff --git a/src/backend/utils/adt/tsquery.c b/src/backend/utils/adt/tsquery.c
index e411a21f07e..008de4236a7 100644
--- a/src/backend/utils/adt/tsquery.c
+++ b/src/backend/utils/adt/tsquery.c
@@ -553,7 +553,7 @@ pushValue_internal(TSQueryParserState state, pg_crc32 valcrc, int distance, int
 				(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
 				 errmsg("value is too big in tsquery: \"%s\"",
 						state->buffer)));
-	if (lenval >= MAXSTRLEN)
+	if (lenval > MAXSTRLEN)
 		ereturn(state->escontext,,
 				(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
 				 errmsg("operand is too long in tsquery: \"%s\"",
@@ -581,7 +581,7 @@ pushValue(TSQueryParserState state, char *strval, int lenval, int16 weight, bool
 {
 	pg_crc32	valcrc;
 
-	if (lenval >= MAXSTRLEN)
+	if (lenval > MAXSTRLEN)
 		ereturn(state->escontext,,
 				(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
 				 errmsg("word is too long in tsquery: \"%s\"",
diff --git a/src/backend/utils/adt/tsvector.c b/src/backend/utils/adt/tsvector.c
index 40be82b890a..b99878d0ca2 100644
--- a/src/backend/utils/adt/tsvector.c
+++ b/src/backend/utils/adt/tsvector.c
@@ -207,12 +207,12 @@ tsvectorin(PG_FUNCTION_ARGS)
 
 	while (gettoken_tsvector(state, &token, &toklen, &pos, &poslen, NULL))
 	{
-		if (toklen >= MAXSTRLEN)
+		if (toklen > MAXSTRLEN)
 			ereturn(escontext, (Datum) 0,
 					(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
 					 errmsg("word is too long (%d bytes, max %d bytes)",
 							toklen,
-							MAXSTRLEN - 1)));
+							MAXSTRLEN)));
 
 		if (cur - tmpbuf > MAXSTRPOS)
 			ereturn(escontext, (Datum) 0,
diff --git a/src/test/regress/expected/tsearch.out b/src/test/regress/expected/tsearch.out
index 5b7c2123f37..52dec59c574 100644
--- a/src/test/regress/expected/tsearch.out
+++ b/src/test/regress/expected/tsearch.out
@@ -1291,6 +1291,21 @@ SELECT length(to_tsvector('english', '345 qwe@efd.r '' http://www.com/ http://ae
      56
 (1 row)
 
+-- a word of exactly the maximum lexeme length is indexed, one byte more is not
+SELECT length((tsvector_to_array(to_tsvector('simple', repeat('x', 2047))))[1]);
+ length 
+--------
+   2047
+(1 row)
+
+SELECT to_tsvector('simple', repeat('x', 2048));
+NOTICE:  word is too long to be indexed
+DETAIL:  Words longer than 2047 characters are ignored.
+ to_tsvector 
+-------------
+ 
+(1 row)
+
 -- ts_debug
 SELECT * from ts_debug('english', '<myns:foo-bar_baz.blurfl>abc&nm1;def&#xa9;ghi&#245;jkl</myns:foo-bar_baz.blurfl>');
    alias   |   description   |           token            |  dictionaries  |  dictionary  | lexemes 
diff --git a/src/test/regress/expected/tstypes.out b/src/test/regress/expected/tstypes.out
index 33d30cd6a09..4d4146d9e25 100644
--- a/src/test/regress/expected/tstypes.out
+++ b/src/test/regress/expected/tstypes.out
@@ -89,6 +89,15 @@ SELECT $$'' '1' '2'$$::tsvector;  -- error, empty lexeme is not allowed
 ERROR:  syntax error in tsvector: "'' '1' '2'"
 LINE 1: SELECT $$'' '1' '2'$$::tsvector;
                ^
+-- a lexeme of exactly the maximum length is allowed, one byte more is not
+SELECT length((tsvector_to_array(repeat('x', 2047)::tsvector))[1]);
+ length 
+--------
+   2047
+(1 row)
+
+SELECT repeat('x', 2048)::tsvector;
+ERROR:  word is too long (2048 bytes, max 2047 bytes)
 -- Also try it with non-error-throwing API
 SELECT pg_input_is_valid('foo', 'tsvector');
  pg_input_is_valid 
@@ -355,6 +364,19 @@ SELECT 'a:* & nbb:*ac | doo:a* | goo'::tsquery;
  'a':* & 'nbb':*AC | 'doo':*A | 'goo'
 (1 row)
 
+-- an operand of exactly the maximum length is allowed, one byte more is not
+SELECT repeat('x', 2047)::tsquery::text = '''' || repeat('x', 2047) || '''';
+ ?column? 
+----------
+ t
+(1 row)
+
+SELECT pg_input_is_valid(repeat('x', 2048), 'tsquery');
+ pg_input_is_valid 
+-------------------
+ f
+(1 row)
+
 SELECT '!!b'::tsquery;
  tsquery 
 ---------
@@ -1386,6 +1408,12 @@ ERROR:  lexeme array may not contain nulls
 SELECT array_to_tsvector(ARRAY['base','hidden','rebel','spaceship', '']);
 ERROR:  lexeme array may not contain empty strings
 -- lexeme of exactly the maximum length is accepted, one byte more is not
+SELECT array_to_tsvector(ARRAY[repeat('x', 2047)]) = repeat('x', 2047)::tsvector;
+ ?column? 
+----------
+ t
+(1 row)
+
 SELECT tsvector_to_array(array_to_tsvector(ARRAY[repeat('x', 2047)])) = ARRAY[repeat('x', 2047)];
  ?column? 
 ----------
diff --git a/src/test/regress/sql/tsearch.sql b/src/test/regress/sql/tsearch.sql
index 8b3d700f57c..de8a8dd600c 100644
--- a/src/test/regress/sql/tsearch.sql
+++ b/src/test/regress/sql/tsearch.sql
@@ -289,6 +289,10 @@ SELECT length(to_tsvector('english', '345 qwe@efd.r '' http://www.com/ http://ae
 /usr/local/fff /awdf/dwqe/4325 rewt/ewr wefjn /wqe-324/ewr gist.h gist.h.c gist.c. readline 4.2 4.2. 4.2, readline-4.2 readline-4.2. 234
 <i <b> wow  < jqw <> qwerty'));
 
+-- a word of exactly the maximum lexeme length is indexed, one byte more is not
+SELECT length((tsvector_to_array(to_tsvector('simple', repeat('x', 2047))))[1]);
+SELECT to_tsvector('simple', repeat('x', 2048));
+
 -- ts_debug
 
 SELECT * from ts_debug('english', '<myns:foo-bar_baz.blurfl>abc&nm1;def&#xa9;ghi&#245;jkl</myns:foo-bar_baz.blurfl>');
diff --git a/src/test/regress/sql/tstypes.sql b/src/test/regress/sql/tstypes.sql
index 0361e359044..b745855ee7d 100644
--- a/src/test/regress/sql/tstypes.sql
+++ b/src/test/regress/sql/tstypes.sql
@@ -18,6 +18,9 @@ SELECT tsvectorin(tsvectorout($$'\\as' ab\c ab\\c AB\\\c ab\\\\c$$::tsvector));
 SELECT '''w'':4A,3B,2C,1D,5 a:8';
 SELECT 'a:3A b:2a'::tsvector || 'ba:1234 a:1B';
 SELECT $$'' '1' '2'$$::tsvector;  -- error, empty lexeme is not allowed
+-- a lexeme of exactly the maximum length is allowed, one byte more is not
+SELECT length((tsvector_to_array(repeat('x', 2047)::tsvector))[1]);
+SELECT repeat('x', 2048)::tsvector;
 
 -- Also try it with non-error-throwing API
 SELECT pg_input_is_valid('foo', 'tsvector');
@@ -66,6 +69,9 @@ SELECT '1&(2&(4&(5|!6)))'::tsquery;
 SELECT E'1&(''2''&('' 4''&(\\|5 | ''6 \\'' !|&'')))'::tsquery;
 SELECT $$'\\as'$$::tsquery;
 SELECT 'a:* & nbb:*ac | doo:a* | goo'::tsquery;
+-- an operand of exactly the maximum length is allowed, one byte more is not
+SELECT repeat('x', 2047)::tsquery::text = '''' || repeat('x', 2047) || '''';
+SELECT pg_input_is_valid(repeat('x', 2048), 'tsquery');
 SELECT '!!b'::tsquery;
 SELECT '!!!b'::tsquery;
 SELECT '!(!b)'::tsquery;
@@ -267,6 +273,7 @@ SELECT array_to_tsvector(ARRAY['base','hidden','rebel','spaceship','strike']);
 SELECT array_to_tsvector(ARRAY['base','hidden','rebel','spaceship', NULL]);
 SELECT array_to_tsvector(ARRAY['base','hidden','rebel','spaceship', '']);
 -- lexeme of exactly the maximum length is accepted, one byte more is not
+SELECT array_to_tsvector(ARRAY[repeat('x', 2047)]) = repeat('x', 2047)::tsvector;
 SELECT tsvector_to_array(array_to_tsvector(ARRAY[repeat('x', 2047)])) = ARRAY[repeat('x', 2047)];
 SELECT array_to_tsvector(ARRAY[repeat('x', 2048)]);
 -- array_to_tsvector must sort and de-dup
-- 
2.54.0

