From 942f374a4383502cbb66d0d83f0ba317962a9a24 Mon Sep 17 00:00:00 2001 From: Jeff Davis Date: Fri, 19 Jun 2026 14:58:47 -0700 Subject: [PATCH v3 4/4] unicode_case.c: use new utf8encode/utf8decode APIs. Reviewed-by: Chao Li Discussion: https://postgr.es/m/c355354e6c3f4a7aafb047361b73db247260fca0.camel@j-davis.com --- src/backend/utils/adt/pg_locale_builtin.c | 17 +++--- src/common/unicode/case_test.c | 2 +- src/common/unicode_case.c | 70 +++++++++++------------ 3 files changed, 44 insertions(+), 45 deletions(-) diff --git a/src/backend/utils/adt/pg_locale_builtin.c b/src/backend/utils/adt/pg_locale_builtin.c index 96da9c6fcf3..a826ec1cfa8 100644 --- a/src/backend/utils/adt/pg_locale_builtin.c +++ b/src/backend/utils/adt/pg_locale_builtin.c @@ -62,25 +62,26 @@ initcap_wbnext(void *state) while (wbstate->offset < wbstate->len) { - int ulen = pg_utf_mblen((const unsigned char *) wbstate->str + - wbstate->offset); + int ulen; char32_t u; bool curr_alnum; + size_t prev_offset = wbstate->offset; - if (wbstate->offset + ulen > wbstate->len) + ulen = utf8decode(&u, (const unsigned char *) wbstate->str + wbstate->offset, + wbstate->len - wbstate->offset); + + /* invalid UTF8 */ + if (ulen <= 0) { + wbstate->init = true; wbstate->offset = wbstate->len; - return wbstate->len; + return prev_offset; } - u = utf8_to_unicode((const unsigned char *) wbstate->str + - wbstate->offset); curr_alnum = pg_u_isalnum(u, wbstate->posix); if (!wbstate->init || curr_alnum != wbstate->prev_alnum) { - size_t prev_offset = wbstate->offset; - wbstate->init = true; wbstate->offset += ulen; wbstate->prev_alnum = curr_alnum; diff --git a/src/common/unicode/case_test.c b/src/common/unicode/case_test.c index ae0f86ffa0c..71a6f2bbe70 100644 --- a/src/common/unicode/case_test.c +++ b/src/common/unicode/case_test.c @@ -179,7 +179,7 @@ test_icu(void) { pg_unicode_category category = unicode_category(code); - if (category != PG_U_UNASSIGNED) + if (category != PG_U_UNASSIGNED && category != PG_U_SURROGATE) { uint8_t icu_category = u_charType(code); char code_str[5] = {0}; diff --git a/src/common/unicode_case.c b/src/common/unicode_case.c index 4a692cfa249..d89f5ca4740 100644 --- a/src/common/unicode_case.c +++ b/src/common/unicode_case.c @@ -194,22 +194,6 @@ unicode_strfold(char *dst, size_t dstsize, const char *src, size_t srclen, NULL, NULL); } -/* local version of pg_utf_mblen() to be inlinable */ -static int -utf8_mblen(const unsigned char *s) -{ - if ((*s & 0x80) == 0) - return 1; - else if ((*s & 0xe0) == 0xc0) - return 2; - else if ((*s & 0xf0) == 0xe0) - return 3; - else if ((*s & 0xf8) == 0xf0) - return 4; - else - return -1; -} - /* * Implement Unicode Default Case Conversion algorithm. * @@ -248,18 +232,19 @@ convert_case(char *dst, size_t dstsize, const char *src, size_t srclen, while (srcoff < srclen) { - int u1len = utf8_mblen((const unsigned char *) src + srcoff); char32_t u1; + int u1len; char32_t simple = 0; const char32_t *special = NULL; enum CaseMapResult casemap_result; + u1len = utf8decode(&u1, (const unsigned char *) src + srcoff, + srclen - srcoff); + /* invalid UTF8 */ - if (u1len < 0 || srcoff + u1len > srclen) + if (u1len <= 0) break; - u1 = utf8_to_unicode((const unsigned char *) src + srcoff); - if (str_casekind == CaseTitle) { if (srcoff == boundary) @@ -280,6 +265,7 @@ convert_case(char *dst, size_t dstsize, const char *src, size_t srclen, /* no mapping; copy bytes from src */ Assert(simple == 0); Assert(special == NULL); + if (result_len + u1len <= dstsize) memcpy(dst + result_len, src + srcoff, u1len); @@ -289,11 +275,18 @@ convert_case(char *dst, size_t dstsize, const char *src, size_t srclen, { /* replace with single character */ char32_t u2 = simple; - char32_t u2len = unicode_utf8len(u2); + int u2len; + size_t remaining = 0; + unsigned char *p = NULL; + + if (dstsize > result_len) + { + remaining = dstsize - result_len; + p = (unsigned char *) dst + result_len; + } Assert(special == NULL); - if (result_len + u2len <= dstsize) - unicode_to_utf8(u2, (unsigned char *) dst + result_len); + u2len = utf8encode(p, remaining, u2); result_len += u2len; } @@ -304,10 +297,17 @@ convert_case(char *dst, size_t dstsize, const char *src, size_t srclen, for (int i = 0; i < MAX_CASE_EXPANSION && special[i]; i++) { char32_t u2 = special[i]; - size_t u2len = unicode_utf8len(u2); + int u2len; + size_t remaining = 0; + unsigned char *p = NULL; - if (result_len + u2len <= dstsize) - unicode_to_utf8(u2, (unsigned char *) dst + result_len); + if (dstsize > result_len) + { + remaining = dstsize - result_len; + p = (unsigned char *) dst + result_len; + } + + u2len = utf8encode(p, remaining, u2); result_len += u2len; } @@ -344,15 +344,15 @@ check_final_sigma(const unsigned char *str, size_t len, size_t offset) { if ((str[i] & 0x80) == 0 || (str[i] & 0xC0) == 0xC0) { - int u1len = utf8_mblen((const unsigned char *) str + i); + int u1len; char32_t curr; + u1len = utf8decode(&curr, (const unsigned char *) str + i, len - i); + /* invalid UTF8 */ - if (u1len < 0 || i + u1len > len) + if (u1len <= 0) return false; - curr = utf8_to_unicode(str + i); - if (pg_u_prop_case_ignorable(curr)) continue; else if (pg_u_prop_cased(curr)) @@ -373,15 +373,15 @@ check_final_sigma(const unsigned char *str, size_t len, size_t offset) { if ((str[i] & 0x80) == 0 || (str[i] & 0xC0) == 0xC0) { - int u1len = utf8_mblen((const unsigned char *) str + i); + int u1len; char32_t curr; + u1len = utf8decode(&curr, (const unsigned char *) str + i, len - i); + /* invalid UTF8 */ - if (u1len < 0 || i + u1len > len) + if (u1len <= 0) return false; - curr = utf8_to_unicode(str + i); - if (pg_u_prop_case_ignorable(curr)) continue; else if (pg_u_prop_cased(curr)) @@ -391,8 +391,6 @@ check_final_sigma(const unsigned char *str, size_t len, size_t offset) } else if ((str[i] & 0xC0) == 0x80) continue; - - Assert(false); /* invalid UTF-8 */ } return true; -- 2.43.0