From 0666256a021ab4d8c8193d23e001a52f75fa893d Mon Sep 17 00:00:00 2001 From: Andrew Dunstan Date: Sun, 13 Sep 2026 17:13:15 -0400 Subject: [PATCH] Add ASCII fast path to Unicode normalization functions unicode_is_normalized(), unicode_assigned(), and unicode_normalize_func() each decoded its input to an array of char32_t codepoints, one utf8_to_unicode()/pg_utf_mblen() call at a time, before doing any real work. A string that is entirely ASCII (code points 0-127) has no canonical or compatibility decomposition anywhere in it, and a combining class of zero throughout, so it is trivially normalized under NFC/NFKC/NFD/NFKD alike. Every one of its code points is assigned too. Detect that case up front with a byte scan using the existing SIMD-vectorized is_valid_ascii() (already used by pg_utf8_verifystr()), and skip the decode loop and per-codepoint work entirely. Add regression coverage for the ASCII-hit case in all three functions, plus a chunk/remainder boundary sweep that plants a non-NFC sequence at varying offsets to catch any off-by-one in text_is_ascii()'s SIMD/ scalar split. --- src/backend/utils/adt/varlena.c | 47 ++++++++++++++++ src/test/regress/expected/unicode.out | 78 +++++++++++++++++++++++++++ src/test/regress/sql/unicode.sql | 14 +++++ 3 files changed, 139 insertions(+) diff --git a/src/backend/utils/adt/varlena.c b/src/backend/utils/adt/varlena.c index f6a41e709ae..1b6013facc2 100644 --- a/src/backend/utils/adt/varlena.c +++ b/src/backend/utils/adt/varlena.c @@ -35,6 +35,7 @@ #include "parser/scansup.h" #include "port/pg_bswap.h" #include "regex/regex.h" +#include "utils/ascii.h" #include "utils/builtins.h" #include "utils/guc.h" #include "utils/lsyscache.h" @@ -167,6 +168,7 @@ static void text_format_string_conversion(StringInfo buf, char conversion, int flags, int width); static void text_format_append_string(StringInfo buf, const char *str, int flags, int width); +static bool text_is_ascii(text *t); /***************************************************************************** @@ -5480,6 +5482,39 @@ icu_unicode_version(PG_FUNCTION_ARGS) PG_RETURN_NULL(); } +/* + * Check whether a text value is pure ASCII. + * + * Pure ASCII (code points 0-127) is unaffected by Unicode normalization + * and is always an assigned code point, independent of server encoding, + * so callers can use this to skip multibyte decoding entirely. This is + * byte-oriented (checking raw bytes for the high bit, rather than + * comparing byte length to codepoint count) so it stays correct if ever + * reused somewhere the server encoding isn't already known to be UTF8. + * + * is_valid_ascii() rejects embedded zero bytes as well as high-bit + * bytes, and requires its length argument to be a multiple of the SIMD + * chunk size (sizeof(Vector8)), so scan the largest chunk-aligned + * prefix with it, then apply the same zero-byte/high-bit check + * byte-at-a-time to the remainder for consistency. + */ +static bool +text_is_ascii(text *t) +{ + unsigned char *s = (unsigned char *) VARDATA_ANY(t); + int len = VARSIZE_ANY_EXHDR(t); + int chunk_len = len - (len % sizeof(Vector8)); + + if (chunk_len > 0 && !is_valid_ascii(s, chunk_len)) + return false; + + for (int i = chunk_len; i < len; i++) + if (s[i] == 0 || IS_HIGHBIT_SET(s[i])) + return false; + + return true; +} + /* * Check whether the string contains only assigned Unicode code * points. Requires that the database encoding is UTF-8. @@ -5495,6 +5530,10 @@ unicode_assigned(PG_FUNCTION_ARGS) ereport(ERROR, (errmsg("Unicode categorization can only be performed if server encoding is UTF8"))); + /* ASCII code points are always assigned */ + if (text_is_ascii(input)) + PG_RETURN_BOOL(true); + /* convert to char32_t */ size = pg_mbstrlen_with_len(VARDATA_ANY(input), VARSIZE_ANY_EXHDR(input)); p = (unsigned char *) VARDATA_ANY(input); @@ -5527,6 +5566,10 @@ unicode_normalize_func(PG_FUNCTION_ARGS) form = unicode_norm_form_from_string(formstr); + /* ASCII code points are unaffected by normalization */ + if (text_is_ascii(input)) + PG_RETURN_TEXT_P(input); + /* convert to char32_t */ size = pg_mbstrlen_with_len(VARDATA_ANY(input), VARSIZE_ANY_EXHDR(input)); input_chars = palloc_array(char32_t, size + 1); @@ -5595,6 +5638,10 @@ unicode_is_normalized(PG_FUNCTION_ARGS) form = unicode_norm_form_from_string(formstr); + /* ASCII code points are always normalized, in any of the four forms */ + if (text_is_ascii(input)) + PG_RETURN_BOOL(true); + /* convert to char32_t */ size = pg_mbstrlen_with_len(VARDATA_ANY(input), VARSIZE_ANY_EXHDR(input)); input_chars = palloc_array(char32_t, size + 1); diff --git a/src/test/regress/expected/unicode.out b/src/test/regress/expected/unicode.out index 63e48d3a961..e716aa7fca2 100644 --- a/src/test/regress/expected/unicode.out +++ b/src/test/regress/expected/unicode.out @@ -26,6 +26,12 @@ SELECT unicode_assigned(U&'abc\+10FFFF'); f (1 row) +SELECT unicode_assigned('abc'); + unicode_assigned +------------------ + t +(1 row) + SELECT normalize(''); normalize ----------- @@ -68,6 +74,19 @@ SELECT normalize(U&'\00E4\24D1c', NFKD) = U&'\0061\0308bc' COLLATE "C" AS test_n t (1 row) +SELECT normalize('abc') = 'abc' COLLATE "C" AS test_ascii_idem; + test_ascii_idem +----------------- + t +(1 row) + +SELECT normalize(val) = val COLLATE "C" AS test_ascii_idem_column +FROM (VALUES ('abc'::text)) v(val); + test_ascii_idem_column +------------------------ + t +(1 row) + SELECT "normalize"('abc', 'def'); -- run-time error ERROR: invalid normalization form: def SELECT U&'\00E4\24D1c' IS NORMALIZED AS test_default; @@ -82,6 +101,12 @@ SELECT U&'\00E4\24D1c' IS NFC NORMALIZED AS test_nfc; t (1 row) +SELECT 'abc' IS NORMALIZED AS test_ascii; + test_ascii +------------ + t +(1 row) + SELECT num, val, val IS NFC NORMALIZED AS NFC, val IS NFD NORMALIZED AS NFD, @@ -105,6 +130,59 @@ ORDER BY num; SELECT is_normalized('abc', 'def'); -- run-time error ERROR: invalid normalization form: def +-- Exercise the ASCII fast-path's chunk/remainder boundary handling: a +-- non-NFC-normalized codepoint sequence must still be detected as such +-- regardless of how much pure-ASCII padding surrounds it. +SELECT len, + (repeat('a', len) || U&'\0061\0308') IS NORMALIZED AS non_nfc_at_end, + (U&'\0061\0308' || repeat('a', len)) IS NORMALIZED AS non_nfc_at_start +FROM generate_series(0, 40) AS len +ORDER BY len; + len | non_nfc_at_end | non_nfc_at_start +-----+----------------+------------------ + 0 | f | f + 1 | f | f + 2 | f | f + 3 | f | f + 4 | f | f + 5 | f | f + 6 | f | f + 7 | f | f + 8 | f | f + 9 | f | f + 10 | f | f + 11 | f | f + 12 | f | f + 13 | f | f + 14 | f | f + 15 | f | f + 16 | f | f + 17 | f | f + 18 | f | f + 19 | f | f + 20 | f | f + 21 | f | f + 22 | f | f + 23 | f | f + 24 | f | f + 25 | f | f + 26 | f | f + 27 | f | f + 28 | f | f + 29 | f | f + 30 | f | f + 31 | f | f + 32 | f | f + 33 | f | f + 34 | f | f + 35 | f | f + 36 | f | f + 37 | f | f + 38 | f | f + 39 | f | f + 40 | f | f +(41 rows) + -- Hangul NFC recomposition tests -- L+V -> LV composition (first and last) SELECT normalize(U&'\1100\1161', NFC) = U&'\AC00' COLLATE "C" AS hangul_lv_first; diff --git a/src/test/regress/sql/unicode.sql b/src/test/regress/sql/unicode.sql index 951f86a336e..e3c030c3744 100644 --- a/src/test/regress/sql/unicode.sql +++ b/src/test/regress/sql/unicode.sql @@ -8,6 +8,7 @@ SELECT U&'\0061\0308bc' <> U&'\00E4bc' COLLATE "C" AS sanity_check; SELECT unicode_version() IS NOT NULL; SELECT unicode_assigned(U&'abc'); SELECT unicode_assigned(U&'abc\+10FFFF'); +SELECT unicode_assigned('abc'); SELECT normalize(''); SELECT normalize(U&'\0061\0308\24D1c') = U&'\00E4\24D1c' COLLATE "C" AS test_default; @@ -16,11 +17,15 @@ SELECT normalize(U&'\00E4bc', NFC) = U&'\00E4bc' COLLATE "C" AS test_nfc_idem; SELECT normalize(U&'\00E4\24D1c', NFD) = U&'\0061\0308\24D1c' COLLATE "C" AS test_nfd; SELECT normalize(U&'\0061\0308\24D1c', NFKC) = U&'\00E4bc' COLLATE "C" AS test_nfkc; SELECT normalize(U&'\00E4\24D1c', NFKD) = U&'\0061\0308bc' COLLATE "C" AS test_nfkd; +SELECT normalize('abc') = 'abc' COLLATE "C" AS test_ascii_idem; +SELECT normalize(val) = val COLLATE "C" AS test_ascii_idem_column +FROM (VALUES ('abc'::text)) v(val); SELECT "normalize"('abc', 'def'); -- run-time error SELECT U&'\00E4\24D1c' IS NORMALIZED AS test_default; SELECT U&'\00E4\24D1c' IS NFC NORMALIZED AS test_nfc; +SELECT 'abc' IS NORMALIZED AS test_ascii; SELECT num, val, val IS NFC NORMALIZED AS NFC, @@ -37,6 +42,15 @@ ORDER BY num; SELECT is_normalized('abc', 'def'); -- run-time error +-- Exercise the ASCII fast-path's chunk/remainder boundary handling: a +-- non-NFC-normalized codepoint sequence must still be detected as such +-- regardless of how much pure-ASCII padding surrounds it. +SELECT len, + (repeat('a', len) || U&'\0061\0308') IS NORMALIZED AS non_nfc_at_end, + (U&'\0061\0308' || repeat('a', len)) IS NORMALIZED AS non_nfc_at_start +FROM generate_series(0, 40) AS len +ORDER BY len; + -- Hangul NFC recomposition tests -- L+V -> LV composition (first and last) SELECT normalize(U&'\1100\1161', NFC) = U&'\AC00' COLLATE "C" AS hangul_lv_first; -- 2.43.0