From 4ad231668103c9e95562fce9253bfa7f94ae58d0 Mon Sep 17 00:00:00 2001 From: Nazir Bilal Yavuz Date: Tue, 15 Sep 2026 15:48:47 +0300 Subject: [PATCH v2 2/2] Return index at text_ascii_check() --- src/backend/utils/adt/varlena.c | 75 +++++++++++++++++++++++---------- 1 file changed, 53 insertions(+), 22 deletions(-) diff --git a/src/backend/utils/adt/varlena.c b/src/backend/utils/adt/varlena.c index 3065b837537..f9eb0e6491f 100644 --- a/src/backend/utils/adt/varlena.c +++ b/src/backend/utils/adt/varlena.c @@ -168,7 +168,7 @@ static void text_format_string_conversion(StringInfo buf, char conversion, int flags, int width); static void text_format_append_string(StringInfo buf, const char *str, int flags, int width); -static bool text_is_ascii(text *t); +static int text_ascii_check(text *t); /***************************************************************************** @@ -5485,6 +5485,10 @@ icu_unicode_version(PG_FUNCTION_ARGS) /* * Check whether a text value is pure ASCII. * + * Return -1 if it is, otherwise the zero-based byte offset of the first + * failing SIMD chunk, or the failing byte in the scalar remainder. All + * bytes before the returned offset are known to be nonzero ASCII. + * * Pure ASCII (code points 0-127) is unaffected by Unicode normalization * and is always an assigned code point, independent of server encoding, * so callers can use this to skip multibyte decoding entirely. This is @@ -5494,25 +5498,26 @@ icu_unicode_version(PG_FUNCTION_ARGS) * * is_valid_ascii() rejects embedded zero bytes as well as high-bit * bytes, and requires its length argument to be a multiple of the SIMD - * chunk size (sizeof(Vector8)), so scan the largest chunk-aligned - * prefix with it, then apply the same zero-byte/high-bit check + * chunk size (sizeof(Vector8)), so check one chunk at a time to locate + * the first failure, then apply the same zero-byte/high-bit check * byte-at-a-time to the remainder for consistency. */ -static bool -text_is_ascii(text *t) +static int +text_ascii_check(text *t) { unsigned char *s = (unsigned char *) VARDATA_ANY(t); int len = VARSIZE_ANY_EXHDR(t); int chunk_len = len - (len % sizeof(Vector8)); - if (chunk_len > 0 && !is_valid_ascii(s, chunk_len)) - return false; + for (int i = 0; i < chunk_len; i += sizeof(Vector8)) + if (!is_valid_ascii(s + i, sizeof(Vector8))) + return i; for (int i = chunk_len; i < len; i++) if (s[i] == 0 || IS_HIGHBIT_SET(s[i])) - return false; + return i; - return true; + return -1; } /* @@ -5525,18 +5530,21 @@ unicode_assigned(PG_FUNCTION_ARGS) text *input = PG_GETARG_TEXT_PP(0); unsigned char *p; int size; + int start; if (GetDatabaseEncoding() != PG_UTF8) ereport(ERROR, (errmsg("Unicode categorization can only be performed if server encoding is UTF8"))); /* ASCII code points are always assigned */ - if (text_is_ascii(input)) + start = text_ascii_check(input); + if (start == -1) PG_RETURN_BOOL(true); - /* convert to char32_t */ - size = pg_mbstrlen_with_len(VARDATA_ANY(input), VARSIZE_ANY_EXHDR(input)); - p = (unsigned char *) VARDATA_ANY(input); + /* Convert only the suffix not already known to be ASCII to char32_t. */ + size = pg_mbstrlen_with_len(VARDATA_ANY(input) + start, + VARSIZE_ANY_EXHDR(input) - start); + p = (unsigned char *) VARDATA_ANY(input) + start; for (int i = 0; i < size; i++) { char32_t uchar = utf8_to_unicode(p); @@ -5563,17 +5571,28 @@ unicode_normalize_func(PG_FUNCTION_ARGS) unsigned char *p; text *result; size_t i; + int start; form = unicode_norm_form_from_string(formstr); /* ASCII code points are unaffected by normalization */ - if (text_is_ascii(input)) + start = text_ascii_check(input); + if (start == -1) PG_RETURN_TEXT_P(input); - /* convert to char32_t */ - size = pg_mbstrlen_with_len(VARDATA_ANY(input), VARSIZE_ANY_EXHDR(input)); + /* + * Keep the last ASCII character in the suffix, since a following + * combining mark could compose with it. ASCII characters have combining + * class zero, so normalization cannot affect any earlier characters. + */ + if (start > 0) + start--; + + /* convert the suffix to char32_t */ + size = pg_mbstrlen_with_len(VARDATA_ANY(input) + start, + VARSIZE_ANY_EXHDR(input) - start); input_chars = palloc_array(char32_t, size + 1); - p = (unsigned char *) VARDATA_ANY(input); + p = (unsigned char *) VARDATA_ANY(input) + start; for (i = 0; i < size; i++) { input_chars[i] = utf8_to_unicode(p); @@ -5586,7 +5605,7 @@ unicode_normalize_func(PG_FUNCTION_ARGS) output_chars = unicode_normalize(form, input_chars); /* convert back to UTF-8 string */ - size = 0; + size = start; for (char32_t *wp = output_chars; *wp; wp++) { unsigned char buf[4]; @@ -5599,6 +5618,8 @@ unicode_normalize_func(PG_FUNCTION_ARGS) SET_VARSIZE(result, size + VARHDRSZ); p = (unsigned char *) VARDATA_ANY(result); + memcpy(p, VARDATA_ANY(input), start); + p += start; for (char32_t *wp = output_chars; *wp; wp++) { unicode_to_utf8(*wp, p); @@ -5635,17 +5656,27 @@ unicode_is_normalized(PG_FUNCTION_ARGS) UnicodeNormalizationQC quickcheck; size_t output_size; bool result; + int start; form = unicode_norm_form_from_string(formstr); /* ASCII code points are always normalized, in any of the four forms */ - if (text_is_ascii(input)) + start = text_ascii_check(input); + if (start == -1) PG_RETURN_BOOL(true); - /* convert to char32_t */ - size = pg_mbstrlen_with_len(VARDATA_ANY(input), VARSIZE_ANY_EXHDR(input)); + /* + * Include the preceding ASCII character for possible composition with a + * following combining mark, as in unicode_normalize_func(). + */ + if (start > 0) + start--; + + /* convert the suffix to char32_t */ + size = pg_mbstrlen_with_len(VARDATA_ANY(input) + start, + VARSIZE_ANY_EXHDR(input) - start); input_chars = palloc_array(char32_t, size + 1); - p = (unsigned char *) VARDATA_ANY(input); + p = (unsigned char *) VARDATA_ANY(input) + start; for (i = 0; i < size; i++) { input_chars[i] = utf8_to_unicode(p); -- 2.47.3