From d7a74c803a42a9df143d7eb56361aa8ec6f3b951 Mon Sep 17 00:00:00 2001 From: Tristan Partin Date: Fri, 11 Sep 2026 05:37:13 +0000 Subject: [PATCH v1] Stop asserting that Unicode normalization consumes its whole input normalize() and IS NORMALIZED determine how many code points their input holds with pg_mbstrlen_with_len(). They then decode exactly that many code points, and assert afterward that this consumed the entire datum. That invariant does not hold because pg_mbstrlen_with_len() stops at the first of "limit" or a NUL. Postgres doesn't allow text types to contain NUL, so reaching this requires work, but src/test/regress/sql/encoding.sql already constructs such values to pin down how the string functions behave on them. It just never passed them to normalize() or IS NORMALIZED, which is why this went unnoticed. Author: Tristan Partin Signed-off-by: Tristan Partin --- src/backend/utils/adt/varlena.c | 4 ++-- src/test/regress/expected/encoding.out | 12 ++++++++++++ src/test/regress/sql/encoding.sql | 2 ++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/backend/utils/adt/varlena.c b/src/backend/utils/adt/varlena.c index f6a41e709ae..962b7857b7d 100644 --- a/src/backend/utils/adt/varlena.c +++ b/src/backend/utils/adt/varlena.c @@ -5537,7 +5537,7 @@ unicode_normalize_func(PG_FUNCTION_ARGS) p += pg_utf_mblen(p); } input_chars[i] = (char32_t) '\0'; - Assert((char *) p == VARDATA_ANY(input) + VARSIZE_ANY_EXHDR(input)); + Assert((char *) p <= VARDATA_ANY(input) + VARSIZE_ANY_EXHDR(input)); /* action */ output_chars = unicode_normalize(form, input_chars); @@ -5605,7 +5605,7 @@ unicode_is_normalized(PG_FUNCTION_ARGS) p += pg_utf_mblen(p); } input_chars[i] = (char32_t) '\0'; - Assert((char *) p == VARDATA_ANY(input) + VARSIZE_ANY_EXHDR(input)); + Assert((char *) p <= VARDATA_ANY(input) + VARSIZE_ANY_EXHDR(input)); /* quick check (see UAX #15) */ quickcheck = unicode_is_normalized_quickcheck(form, input_chars); diff --git a/src/test/regress/expected/encoding.out b/src/test/regress/expected/encoding.out index 0bb72a1df6f..5c06d6daae4 100644 --- a/src/test/regress/expected/encoding.out +++ b/src/test/regress/expected/encoding.out @@ -120,6 +120,18 @@ SELECT regexp_replace(with_nul, '^caf(.)$', '\1') FROM regress_encoding; é (1 row) +SELECT octet_length(normalize(with_nul)) FROM regress_encoding; + octet_length +-------------- + 5 +(1 row) + +SELECT with_nul IS NORMALIZED FROM regress_encoding; + is_normalized +--------------- + t +(1 row) + -- NUL = character SELECT with_nul, reverse(with_nul), reverse(reverse(with_nul)) FROM regress_encoding; with_nul | reverse | reverse diff --git a/src/test/regress/sql/encoding.sql b/src/test/regress/sql/encoding.sql index 26caa93a5d5..d2a649acaa2 100644 --- a/src/test/regress/sql/encoding.sql +++ b/src/test/regress/sql/encoding.sql @@ -57,6 +57,8 @@ SELECT substring(with_nul, 4, 1) FROM regress_encoding; SELECT substring(with_nul, 5, 1) FROM regress_encoding; SELECT convert_to(substring(with_nul, 5, 1), 'UTF8') FROM regress_encoding; SELECT regexp_replace(with_nul, '^caf(.)$', '\1') FROM regress_encoding; +SELECT octet_length(normalize(with_nul)) FROM regress_encoding; +SELECT with_nul IS NORMALIZED FROM regress_encoding; -- NUL = character SELECT with_nul, reverse(with_nul), reverse(reverse(with_nul)) FROM regress_encoding; -- Tristan Partin https://tristan.partin.io