From 341e3f31d22b5609584cccb8b7def798cc94c983 Mon Sep 17 00:00:00 2001 From: Jeff Davis Date: Thu, 26 Sep 2024 14:30:07 -0700 Subject: [PATCH v7 5/9] Control ctype behavior with a method table. Previously, ctype behavior (pattern matching) behavior branched based on the provider. A method table is less error-prone and easier to hook. --- src/backend/regex/regc_pg_locale.c | 388 +++++-------------------- src/backend/utils/adt/like.c | 22 +- src/backend/utils/adt/like_support.c | 7 +- src/backend/utils/adt/pg_locale.c | 101 +++++++ src/backend/utils/adt/pg_locale_icu.c | 52 ++++ src/backend/utils/adt/pg_locale_libc.c | 158 ++++++++++ src/include/utils/pg_locale.h | 46 +++ src/tools/pgindent/typedefs.list | 1 - 8 files changed, 448 insertions(+), 327 deletions(-) diff --git a/src/backend/regex/regc_pg_locale.c b/src/backend/regex/regc_pg_locale.c index b75784b6ce..e898634fdf 100644 --- a/src/backend/regex/regc_pg_locale.c +++ b/src/backend/regex/regc_pg_locale.c @@ -63,33 +63,18 @@ * NB: the coding here assumes pg_wchar is an unsigned type. */ -typedef enum -{ - PG_REGEX_STRATEGY_C, /* C locale (encoding independent) */ - PG_REGEX_STRATEGY_BUILTIN, /* built-in Unicode semantics */ - PG_REGEX_STRATEGY_LIBC_WIDE, /* Use locale_t functions */ - PG_REGEX_STRATEGY_LIBC_1BYTE, /* Use locale_t functions */ - PG_REGEX_STRATEGY_ICU, /* Use ICU uchar.h functions */ -} PG_Locale_Strategy; - -static PG_Locale_Strategy pg_regex_strategy; static pg_locale_t pg_regex_locale; static Oid pg_regex_collation; +static struct pg_locale_struct dummy_c_locale = { + .collate_is_c = true, + .ctype_is_c = true, +}; + /* * Hard-wired character properties for C locale */ -#define PG_ISDIGIT 0x01 -#define PG_ISALPHA 0x02 -#define PG_ISALNUM (PG_ISDIGIT | PG_ISALPHA) -#define PG_ISUPPER 0x04 -#define PG_ISLOWER 0x08 -#define PG_ISGRAPH 0x10 -#define PG_ISPRINT 0x20 -#define PG_ISPUNCT 0x40 -#define PG_ISSPACE 0x80 - -static const unsigned char pg_char_properties[128] = { +static const unsigned char char_properties_tbl[128] = { /* NUL */ 0, /* ^A */ 0, /* ^B */ 0, @@ -232,7 +217,6 @@ void pg_set_regex_collation(Oid collation) { pg_locale_t locale = 0; - PG_Locale_Strategy strategy; if (!OidIsValid(collation)) { @@ -253,8 +237,8 @@ pg_set_regex_collation(Oid collation) * catalog access is available, so we can't call * pg_newlocale_from_collation(). */ - strategy = PG_REGEX_STRATEGY_C; collation = C_COLLATION_OID; + locale = &dummy_c_locale; } else { @@ -271,32 +255,11 @@ pg_set_regex_collation(Oid collation) * C/POSIX collations use this path regardless of database * encoding */ - strategy = PG_REGEX_STRATEGY_C; - locale = 0; + locale = &dummy_c_locale; collation = C_COLLATION_OID; } - else if (locale->provider == COLLPROVIDER_BUILTIN) - { - Assert(GetDatabaseEncoding() == PG_UTF8); - strategy = PG_REGEX_STRATEGY_BUILTIN; - } -#ifdef USE_ICU - else if (locale->provider == COLLPROVIDER_ICU) - { - strategy = PG_REGEX_STRATEGY_ICU; - } -#endif - else - { - Assert(locale->provider == COLLPROVIDER_LIBC); - if (GetDatabaseEncoding() == PG_UTF8) - strategy = PG_REGEX_STRATEGY_LIBC_WIDE; - else - strategy = PG_REGEX_STRATEGY_LIBC_1BYTE; - } } - pg_regex_strategy = strategy; pg_regex_locale = locale; pg_regex_collation = collation; } @@ -304,82 +267,31 @@ pg_set_regex_collation(Oid collation) static int pg_wc_isdigit(pg_wchar c) { - switch (pg_regex_strategy) - { - case PG_REGEX_STRATEGY_C: - return (c <= (pg_wchar) 127 && - (pg_char_properties[c] & PG_ISDIGIT)); - case PG_REGEX_STRATEGY_BUILTIN: - return pg_u_isdigit(c, true); - case PG_REGEX_STRATEGY_LIBC_WIDE: - if (sizeof(wchar_t) >= 4 || c <= (pg_wchar) 0xFFFF) - return iswdigit_l((wint_t) c, pg_regex_locale->info.lt); - /* FALL THRU */ - case PG_REGEX_STRATEGY_LIBC_1BYTE: - return (c <= (pg_wchar) UCHAR_MAX && - isdigit_l((unsigned char) c, pg_regex_locale->info.lt)); - break; - case PG_REGEX_STRATEGY_ICU: -#ifdef USE_ICU - return u_isdigit(c); -#endif - break; - } - return 0; /* can't get here, but keep compiler quiet */ + if (pg_regex_locale->ctype_is_c) + return (c <= (pg_wchar) 127 && + (char_properties_tbl[c] & PG_ISDIGIT)); + else + return char_properties(c, PG_ISDIGIT, pg_regex_locale) != 0; } static int pg_wc_isalpha(pg_wchar c) { - switch (pg_regex_strategy) - { - case PG_REGEX_STRATEGY_C: - return (c <= (pg_wchar) 127 && - (pg_char_properties[c] & PG_ISALPHA)); - case PG_REGEX_STRATEGY_BUILTIN: - return pg_u_isalpha(c); - case PG_REGEX_STRATEGY_LIBC_WIDE: - if (sizeof(wchar_t) >= 4 || c <= (pg_wchar) 0xFFFF) - return iswalpha_l((wint_t) c, pg_regex_locale->info.lt); - /* FALL THRU */ - case PG_REGEX_STRATEGY_LIBC_1BYTE: - return (c <= (pg_wchar) UCHAR_MAX && - isalpha_l((unsigned char) c, pg_regex_locale->info.lt)); - break; - case PG_REGEX_STRATEGY_ICU: -#ifdef USE_ICU - return u_isalpha(c); -#endif - break; - } - return 0; /* can't get here, but keep compiler quiet */ + if (pg_regex_locale->ctype_is_c) + return (c <= (pg_wchar) 127 && + (char_properties_tbl[c] & PG_ISALPHA)); + else + return char_properties(c, PG_ISALPHA, pg_regex_locale) != 0; } static int pg_wc_isalnum(pg_wchar c) { - switch (pg_regex_strategy) - { - case PG_REGEX_STRATEGY_C: - return (c <= (pg_wchar) 127 && - (pg_char_properties[c] & PG_ISALNUM)); - case PG_REGEX_STRATEGY_BUILTIN: - return pg_u_isalnum(c, true); - case PG_REGEX_STRATEGY_LIBC_WIDE: - if (sizeof(wchar_t) >= 4 || c <= (pg_wchar) 0xFFFF) - return iswalnum_l((wint_t) c, pg_regex_locale->info.lt); - /* FALL THRU */ - case PG_REGEX_STRATEGY_LIBC_1BYTE: - return (c <= (pg_wchar) UCHAR_MAX && - isalnum_l((unsigned char) c, pg_regex_locale->info.lt)); - break; - case PG_REGEX_STRATEGY_ICU: -#ifdef USE_ICU - return u_isalnum(c); -#endif - break; - } - return 0; /* can't get here, but keep compiler quiet */ + if (pg_regex_locale->ctype_is_c) + return (c <= (pg_wchar) 127 && + (char_properties_tbl[c] & PG_ISALNUM)); + else + return char_properties(c, PG_ISDIGIT | PG_ISALPHA, pg_regex_locale) != 0; } static int @@ -394,219 +306,87 @@ pg_wc_isword(pg_wchar c) static int pg_wc_isupper(pg_wchar c) { - switch (pg_regex_strategy) - { - case PG_REGEX_STRATEGY_C: - return (c <= (pg_wchar) 127 && - (pg_char_properties[c] & PG_ISUPPER)); - case PG_REGEX_STRATEGY_BUILTIN: - return pg_u_isupper(c); - case PG_REGEX_STRATEGY_LIBC_WIDE: - if (sizeof(wchar_t) >= 4 || c <= (pg_wchar) 0xFFFF) - return iswupper_l((wint_t) c, pg_regex_locale->info.lt); - /* FALL THRU */ - case PG_REGEX_STRATEGY_LIBC_1BYTE: - return (c <= (pg_wchar) UCHAR_MAX && - isupper_l((unsigned char) c, pg_regex_locale->info.lt)); - break; - case PG_REGEX_STRATEGY_ICU: -#ifdef USE_ICU - return u_isupper(c); -#endif - break; - } - return 0; /* can't get here, but keep compiler quiet */ + if (pg_regex_locale->ctype_is_c) + return (c <= (pg_wchar) 127 && + (char_properties_tbl[c] & PG_ISUPPER)); + else + return char_properties(c, PG_ISUPPER, pg_regex_locale) != 0; } static int pg_wc_islower(pg_wchar c) { - switch (pg_regex_strategy) - { - case PG_REGEX_STRATEGY_C: - return (c <= (pg_wchar) 127 && - (pg_char_properties[c] & PG_ISLOWER)); - case PG_REGEX_STRATEGY_BUILTIN: - return pg_u_islower(c); - case PG_REGEX_STRATEGY_LIBC_WIDE: - if (sizeof(wchar_t) >= 4 || c <= (pg_wchar) 0xFFFF) - return iswlower_l((wint_t) c, pg_regex_locale->info.lt); - /* FALL THRU */ - case PG_REGEX_STRATEGY_LIBC_1BYTE: - return (c <= (pg_wchar) UCHAR_MAX && - islower_l((unsigned char) c, pg_regex_locale->info.lt)); - break; - case PG_REGEX_STRATEGY_ICU: -#ifdef USE_ICU - return u_islower(c); -#endif - break; - } - return 0; /* can't get here, but keep compiler quiet */ + if (pg_regex_locale->ctype_is_c) + return (c <= (pg_wchar) 127 && + (char_properties_tbl[c] & PG_ISLOWER)); + else + return char_properties(c, PG_ISLOWER, pg_regex_locale) != 0; } static int pg_wc_isgraph(pg_wchar c) { - switch (pg_regex_strategy) - { - case PG_REGEX_STRATEGY_C: - return (c <= (pg_wchar) 127 && - (pg_char_properties[c] & PG_ISGRAPH)); - case PG_REGEX_STRATEGY_BUILTIN: - return pg_u_isgraph(c); - case PG_REGEX_STRATEGY_LIBC_WIDE: - if (sizeof(wchar_t) >= 4 || c <= (pg_wchar) 0xFFFF) - return iswgraph_l((wint_t) c, pg_regex_locale->info.lt); - /* FALL THRU */ - case PG_REGEX_STRATEGY_LIBC_1BYTE: - return (c <= (pg_wchar) UCHAR_MAX && - isgraph_l((unsigned char) c, pg_regex_locale->info.lt)); - break; - case PG_REGEX_STRATEGY_ICU: -#ifdef USE_ICU - return u_isgraph(c); -#endif - break; - } - return 0; /* can't get here, but keep compiler quiet */ + if (pg_regex_locale->ctype_is_c) + return (c <= (pg_wchar) 127 && + (char_properties_tbl[c] & PG_ISGRAPH)); + else + return char_properties(c, PG_ISGRAPH, pg_regex_locale) != 0; } static int pg_wc_isprint(pg_wchar c) { - switch (pg_regex_strategy) - { - case PG_REGEX_STRATEGY_C: - return (c <= (pg_wchar) 127 && - (pg_char_properties[c] & PG_ISPRINT)); - case PG_REGEX_STRATEGY_BUILTIN: - return pg_u_isprint(c); - case PG_REGEX_STRATEGY_LIBC_WIDE: - if (sizeof(wchar_t) >= 4 || c <= (pg_wchar) 0xFFFF) - return iswprint_l((wint_t) c, pg_regex_locale->info.lt); - /* FALL THRU */ - case PG_REGEX_STRATEGY_LIBC_1BYTE: - return (c <= (pg_wchar) UCHAR_MAX && - isprint_l((unsigned char) c, pg_regex_locale->info.lt)); - break; - case PG_REGEX_STRATEGY_ICU: -#ifdef USE_ICU - return u_isprint(c); -#endif - break; - } - return 0; /* can't get here, but keep compiler quiet */ + if (pg_regex_locale->ctype_is_c) + return (c <= (pg_wchar) 127 && + (char_properties_tbl[c] & PG_ISPRINT)); + else + return char_properties(c, PG_ISPRINT, pg_regex_locale) != 0; } static int pg_wc_ispunct(pg_wchar c) { - switch (pg_regex_strategy) - { - case PG_REGEX_STRATEGY_C: - return (c <= (pg_wchar) 127 && - (pg_char_properties[c] & PG_ISPUNCT)); - case PG_REGEX_STRATEGY_BUILTIN: - return pg_u_ispunct(c, true); - case PG_REGEX_STRATEGY_LIBC_WIDE: - if (sizeof(wchar_t) >= 4 || c <= (pg_wchar) 0xFFFF) - return iswpunct_l((wint_t) c, pg_regex_locale->info.lt); - /* FALL THRU */ - case PG_REGEX_STRATEGY_LIBC_1BYTE: - return (c <= (pg_wchar) UCHAR_MAX && - ispunct_l((unsigned char) c, pg_regex_locale->info.lt)); - break; - case PG_REGEX_STRATEGY_ICU: -#ifdef USE_ICU - return u_ispunct(c); -#endif - break; - } - return 0; /* can't get here, but keep compiler quiet */ + if (pg_regex_locale->ctype_is_c) + return (c <= (pg_wchar) 127 && + (char_properties_tbl[c] & PG_ISPUNCT)); + else + return char_properties(c, PG_ISPUNCT, pg_regex_locale) != 0; } static int pg_wc_isspace(pg_wchar c) { - switch (pg_regex_strategy) - { - case PG_REGEX_STRATEGY_C: - return (c <= (pg_wchar) 127 && - (pg_char_properties[c] & PG_ISSPACE)); - case PG_REGEX_STRATEGY_BUILTIN: - return pg_u_isspace(c); - case PG_REGEX_STRATEGY_LIBC_WIDE: - if (sizeof(wchar_t) >= 4 || c <= (pg_wchar) 0xFFFF) - return iswspace_l((wint_t) c, pg_regex_locale->info.lt); - /* FALL THRU */ - case PG_REGEX_STRATEGY_LIBC_1BYTE: - return (c <= (pg_wchar) UCHAR_MAX && - isspace_l((unsigned char) c, pg_regex_locale->info.lt)); - break; - case PG_REGEX_STRATEGY_ICU: -#ifdef USE_ICU - return u_isspace(c); -#endif - break; - } - return 0; /* can't get here, but keep compiler quiet */ + if (pg_regex_locale->ctype_is_c) + return (c <= (pg_wchar) 127 && + (char_properties_tbl[c] & PG_ISSPACE)); + else + return char_properties(c, PG_ISSPACE, pg_regex_locale) != 0; } static pg_wchar pg_wc_toupper(pg_wchar c) { - switch (pg_regex_strategy) + if (pg_regex_locale->ctype_is_c) { - case PG_REGEX_STRATEGY_C: - if (c <= (pg_wchar) 127) - return pg_ascii_toupper((unsigned char) c); - return c; - case PG_REGEX_STRATEGY_BUILTIN: - return unicode_uppercase_simple(c); - case PG_REGEX_STRATEGY_LIBC_WIDE: - if (sizeof(wchar_t) >= 4 || c <= (pg_wchar) 0xFFFF) - return towupper_l((wint_t) c, pg_regex_locale->info.lt); - /* FALL THRU */ - case PG_REGEX_STRATEGY_LIBC_1BYTE: - if (c <= (pg_wchar) UCHAR_MAX) - return toupper_l((unsigned char) c, pg_regex_locale->info.lt); - return c; - case PG_REGEX_STRATEGY_ICU: -#ifdef USE_ICU - return u_toupper(c); -#endif - break; + if (c <= (pg_wchar) 127) + return pg_ascii_toupper((unsigned char) c); + return c; } - return 0; /* can't get here, but keep compiler quiet */ + else + return pg_regex_locale->ctype->wc_toupper(c, pg_regex_locale); } static pg_wchar pg_wc_tolower(pg_wchar c) { - switch (pg_regex_strategy) + if (pg_regex_locale->ctype_is_c) { - case PG_REGEX_STRATEGY_C: - if (c <= (pg_wchar) 127) - return pg_ascii_tolower((unsigned char) c); - return c; - case PG_REGEX_STRATEGY_BUILTIN: - return unicode_lowercase_simple(c); - case PG_REGEX_STRATEGY_LIBC_WIDE: - if (sizeof(wchar_t) >= 4 || c <= (pg_wchar) 0xFFFF) - return towlower_l((wint_t) c, pg_regex_locale->info.lt); - /* FALL THRU */ - case PG_REGEX_STRATEGY_LIBC_1BYTE: - if (c <= (pg_wchar) UCHAR_MAX) - return tolower_l((unsigned char) c, pg_regex_locale->info.lt); - return c; - case PG_REGEX_STRATEGY_ICU: -#ifdef USE_ICU - return u_tolower(c); -#endif - break; + if (c <= (pg_wchar) 127) + return pg_ascii_tolower((unsigned char) c); + return c; } - return 0; /* can't get here, but keep compiler quiet */ + else + return pg_regex_locale->ctype->wc_tolower(c, pg_regex_locale); } @@ -732,37 +512,25 @@ pg_ctype_get_cache(pg_wc_probefunc probefunc, int cclasscode) * would always be true for production values of MAX_SIMPLE_CHR, but it's * useful to allow it to be small for testing purposes.) */ - switch (pg_regex_strategy) + if (pg_regex_locale->ctype_is_c) { - case PG_REGEX_STRATEGY_C: #if MAX_SIMPLE_CHR >= 127 - max_chr = (pg_wchar) 127; - pcc->cv.cclasscode = -1; + max_chr = (pg_wchar) 127; + pcc->cv.cclasscode = -1; #else - max_chr = (pg_wchar) MAX_SIMPLE_CHR; + max_chr = (pg_wchar) MAX_SIMPLE_CHR; #endif - break; - case PG_REGEX_STRATEGY_BUILTIN: - max_chr = (pg_wchar) MAX_SIMPLE_CHR; - break; - case PG_REGEX_STRATEGY_LIBC_WIDE: - max_chr = (pg_wchar) MAX_SIMPLE_CHR; - break; - case PG_REGEX_STRATEGY_LIBC_1BYTE: -#if MAX_SIMPLE_CHR >= UCHAR_MAX - max_chr = (pg_wchar) UCHAR_MAX; + } + else + { + if (pg_regex_locale->ctype->max_chr != 0 && + pg_regex_locale->ctype->max_chr <= MAX_SIMPLE_CHR) + { + max_chr = pg_regex_locale->ctype->max_chr; pcc->cv.cclasscode = -1; -#else - max_chr = (pg_wchar) MAX_SIMPLE_CHR; -#endif - break; - case PG_REGEX_STRATEGY_ICU: + } + else max_chr = (pg_wchar) MAX_SIMPLE_CHR; - break; - default: - Assert(false); - max_chr = 0; /* can't get here, but keep compiler quiet */ - break; } /* diff --git a/src/backend/utils/adt/like.c b/src/backend/utils/adt/like.c index 0152723b2a..5b679bcad8 100644 --- a/src/backend/utils/adt/like.c +++ b/src/backend/utils/adt/like.c @@ -96,7 +96,7 @@ SB_lower_char(unsigned char c, pg_locale_t locale) if (locale->ctype_is_c) return pg_ascii_tolower(c); else - return tolower_l(c, locale->info.lt); + return char_tolower(c, locale); } @@ -201,7 +201,17 @@ Generic_Text_IC_like(text *str, text *pat, Oid collation) * way. */ - if (pg_database_encoding_max_length() > 1 || (locale->provider == COLLPROVIDER_ICU)) + if (locale->ctype_is_c || + (char_tolower_enabled(locale) && + pg_database_encoding_max_length() == 1)) + { + p = VARDATA_ANY(pat); + plen = VARSIZE_ANY_EXHDR(pat); + s = VARDATA_ANY(str); + slen = VARSIZE_ANY_EXHDR(str); + return SB_IMatchText(s, slen, p, plen, locale); + } + else { pat = DatumGetTextPP(DirectFunctionCall1Coll(lower, collation, PointerGetDatum(pat))); @@ -216,14 +226,6 @@ Generic_Text_IC_like(text *str, text *pat, Oid collation) else return MB_MatchText(s, slen, p, plen, 0); } - else - { - p = VARDATA_ANY(pat); - plen = VARSIZE_ANY_EXHDR(pat); - s = VARDATA_ANY(str); - slen = VARSIZE_ANY_EXHDR(str); - return SB_IMatchText(s, slen, p, plen, locale); - } } /* diff --git a/src/backend/utils/adt/like_support.c b/src/backend/utils/adt/like_support.c index 8b15509a3b..bf718f1a3d 100644 --- a/src/backend/utils/adt/like_support.c +++ b/src/backend/utils/adt/like_support.c @@ -1498,13 +1498,8 @@ pattern_char_isalpha(char c, bool is_multibyte, { if (locale->ctype_is_c) return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z'); - else if (is_multibyte && IS_HIGHBIT_SET(c)) - return true; - else if (locale->provider != COLLPROVIDER_LIBC) - return IS_HIGHBIT_SET(c) || - (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z'); else - return isalpha_l((unsigned char) c, locale->info.lt); + return char_is_cased(c, locale); } diff --git a/src/backend/utils/adt/pg_locale.c b/src/backend/utils/adt/pg_locale.c index f9a8edf3e0..7b99d00304 100644 --- a/src/backend/utils/adt/pg_locale.c +++ b/src/backend/utils/adt/pg_locale.c @@ -244,6 +244,58 @@ static const struct casemap_methods casemap_methods_builtin = { .strupper = strupper_builtin, }; +static int +char_properties_builtin(pg_wchar wc, int mask, pg_locale_t locale) +{ + int result = 0; + + if ((mask & PG_ISDIGIT) && pg_u_isdigit(wc, true)) + result |= PG_ISDIGIT; + if ((mask & PG_ISALPHA) && pg_u_isalpha(wc)) + result |= PG_ISALPHA; + if ((mask & PG_ISUPPER) && pg_u_isupper(wc)) + result |= PG_ISUPPER; + if ((mask & PG_ISLOWER) && pg_u_islower(wc)) + result |= PG_ISLOWER; + if ((mask & PG_ISGRAPH) && pg_u_isgraph(wc)) + result |= PG_ISGRAPH; + if ((mask & PG_ISPRINT) && pg_u_isprint(wc)) + result |= PG_ISPRINT; + if ((mask & PG_ISPUNCT) && pg_u_ispunct(wc, true)) + result |= PG_ISPUNCT; + if ((mask & PG_ISSPACE) && pg_u_isspace(wc)) + result |= PG_ISSPACE; + + return result; +} + +static bool +char_is_cased_builtin(char ch, pg_locale_t locale) +{ + return IS_HIGHBIT_SET(ch) || + (ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z'); +} + +static pg_wchar +wc_toupper_builtin(pg_wchar wc, pg_locale_t locale) +{ + return unicode_uppercase_simple(wc); +} + +static pg_wchar +wc_tolower_builtin(pg_wchar wc, pg_locale_t locale) +{ + return unicode_lowercase_simple(wc); +} + +static const struct ctype_methods ctype_methods_builtin = { + .char_properties = char_properties_builtin, + .char_is_cased = char_is_cased_builtin, + .wc_tolower = wc_tolower_builtin, + .wc_toupper = wc_toupper_builtin, +}; + + /* * pg_perm_setlocale * @@ -1317,6 +1369,8 @@ create_pg_locale_builtin(Oid collid, MemoryContext context) result->collate_is_c = true; result->ctype_is_c = (strcmp(locstr, "C") == 0); result->casemap = &casemap_methods_builtin; + if (!result->ctype_is_c) + result->ctype = &ctype_methods_builtin; return result; } @@ -1747,6 +1801,53 @@ pg_strnxfrm_prefix(char *dest, size_t destsize, const char *src, return locale->collate->strnxfrm_prefix(dest, destsize, src, srclen, locale); } +/* + * char_properties() + * + * Out of the properties specified in the given mask, return a new mask of the + * properties true for the given character. + */ +int +char_properties(pg_wchar wc, int mask, pg_locale_t locale) +{ + return locale->ctype->char_properties(wc, mask, locale); +} + +/* + * char_is_cased() + * + * Fuzzy test of whether the given char is case-varying or not. The argument + * is a single byte, so in a multibyte encoding, just assume any non-ASCII + * char is case-varying. + */ +bool +char_is_cased(char ch, pg_locale_t locale) +{ + return locale->ctype->char_is_cased(ch, locale); +} + +/* + * char_tolower_enabled() + * + * Does the provider support char_tolower()? + */ +bool +char_tolower_enabled(pg_locale_t locale) +{ + return (locale->ctype->char_tolower != NULL); +} + +/* + * char_tolower() + * + * Convert char (single-byte encoding) to lowercase. + */ +char +char_tolower(unsigned char ch, pg_locale_t locale) +{ + return locale->ctype->char_tolower(ch, locale); +} + /* * Return required encoding ID for the given locale, or -1 if any encoding is * valid for the locale. diff --git a/src/backend/utils/adt/pg_locale_icu.c b/src/backend/utils/adt/pg_locale_icu.c index 3f4ada38bd..95230179a1 100644 --- a/src/backend/utils/adt/pg_locale_icu.c +++ b/src/backend/utils/adt/pg_locale_icu.c @@ -109,6 +109,50 @@ static int32_t u_strToTitle_default_BI(UChar *dest, int32_t destCapacity, const char *locale, UErrorCode *pErrorCode); +static int +char_properties_icu(pg_wchar wc, int mask, pg_locale_t locale) +{ + int result = 0; + + if ((mask & PG_ISDIGIT) && u_isdigit(wc)) + result |= PG_ISDIGIT; + if ((mask & PG_ISALPHA) && u_isalpha(wc)) + result |= PG_ISALPHA; + if ((mask & PG_ISUPPER) && u_isupper(wc)) + result |= PG_ISUPPER; + if ((mask & PG_ISLOWER) && u_islower(wc)) + result |= PG_ISLOWER; + if ((mask & PG_ISGRAPH) && u_isgraph(wc)) + result |= PG_ISGRAPH; + if ((mask & PG_ISPRINT) && u_isprint(wc)) + result |= PG_ISPRINT; + if ((mask & PG_ISPUNCT) && u_ispunct(wc)) + result |= PG_ISPUNCT; + if ((mask & PG_ISSPACE) && u_isspace(wc)) + result |= PG_ISSPACE; + + return result; +} + +static bool +char_is_cased_icu(char ch, pg_locale_t locale) +{ + return IS_HIGHBIT_SET(ch) || + (ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z'); +} + +static pg_wchar +toupper_icu(pg_wchar wc, pg_locale_t locale) +{ + return u_toupper(wc); +} + +static pg_wchar +tolower_icu(pg_wchar wc, pg_locale_t locale) +{ + return u_tolower(wc); +} + static const struct collate_methods collate_methods_icu = { .strncoll = strncoll_icu, .strnxfrm = strnxfrm_icu, @@ -132,6 +176,13 @@ static const struct casemap_methods casemap_methods_icu = { .strtitle = strtitle_icu, .strupper = strupper_icu, }; + +static const struct ctype_methods ctype_methods_icu = { + .char_properties = char_properties_icu, + .char_is_cased = char_is_cased_icu, + .wc_toupper = toupper_icu, + .wc_tolower = tolower_icu, +}; #endif pg_locale_t @@ -203,6 +254,7 @@ create_pg_locale_icu(Oid collid, MemoryContext context) else result->collate = &collate_methods_icu; result->casemap = &casemap_methods_icu; + result->ctype = &ctype_methods_icu; return result; #else diff --git a/src/backend/utils/adt/pg_locale_libc.c b/src/backend/utils/adt/pg_locale_libc.c index 798903ed2b..d65403af8d 100644 --- a/src/backend/utils/adt/pg_locale_libc.c +++ b/src/backend/utils/adt/pg_locale_libc.c @@ -70,6 +70,15 @@ static size_t strupper_libc_mb(char *dest, size_t destsize, const char *src, ssize_t srclen, pg_locale_t locale); +static int char_properties_libc_1byte(pg_wchar wc, int mask, + pg_locale_t locale); +static int char_properties_libc_wide(pg_wchar wc, int mask, + pg_locale_t locale); +static pg_wchar toupper_libc_1byte(pg_wchar wc, pg_locale_t locale); +static pg_wchar toupper_libc_wide(pg_wchar wc, pg_locale_t locale); +static pg_wchar tolower_libc_1byte(pg_wchar wc, pg_locale_t locale); +static pg_wchar tolower_libc_wide(pg_wchar wc, pg_locale_t locale); + static const struct collate_methods collate_methods_libc = { .strncoll = strncoll_libc, .strnxfrm = strnxfrm_libc, @@ -104,6 +113,24 @@ static const struct collate_methods collate_methods_libc_win32_utf8 = { }; #endif +static bool +char_is_cased_libc(char ch, pg_locale_t locale) +{ + bool is_multibyte = pg_database_encoding_max_length() > 1; + + if (is_multibyte && IS_HIGHBIT_SET(ch)) + return true; + else + return isalpha_l((unsigned char) ch, locale->info.lt); +} + +static char +char_tolower_libc(unsigned char ch, pg_locale_t locale) +{ + Assert(pg_database_encoding_max_length() == 1); + return tolower_l(ch, locale->info.lt); +} + static const struct casemap_methods casemap_methods_libc_sb = { .strlower = strlower_libc_sb, .strtitle = strtitle_libc_sb, @@ -116,6 +143,23 @@ static const struct casemap_methods casemap_methods_libc_mb = { .strupper = strupper_libc_mb, }; +static const struct ctype_methods ctype_methods_libc_1byte = { + .char_properties = char_properties_libc_1byte, + .char_is_cased = char_is_cased_libc, + .char_tolower = char_tolower_libc, + .wc_toupper = toupper_libc_1byte, + .wc_tolower = tolower_libc_1byte, + .max_chr = UCHAR_MAX, +}; + +static const struct ctype_methods ctype_methods_libc_wide = { + .char_properties = char_properties_libc_wide, + .char_is_cased = char_is_cased_libc, + .char_tolower = char_tolower_libc, + .wc_toupper = toupper_libc_wide, + .wc_tolower = tolower_libc_wide, +}; + static size_t strlower_libc_sb(char *dest, size_t destsize, const char *src, ssize_t srclen, pg_locale_t locale) @@ -435,6 +479,13 @@ create_pg_locale_libc(Oid collid, MemoryContext context) else result->casemap = &casemap_methods_libc_sb; } + if (!result->ctype_is_c) + { + if (GetDatabaseEncoding() == PG_UTF8) + result->ctype = &ctype_methods_libc_wide; + else + result->ctype = &ctype_methods_libc_1byte; + } return result; } @@ -718,6 +769,113 @@ report_newlocale_failure(const char *localename) localename) : 0))); } +static int +char_properties_libc_1byte(pg_wchar wc, int mask, pg_locale_t locale) +{ + int result = 0; + + Assert(!locale->ctype_is_c); + Assert(GetDatabaseEncoding() != PG_UTF8); + + if (wc > (pg_wchar) UCHAR_MAX) + return 0; + + if ((mask & PG_ISDIGIT) && isdigit_l((unsigned char) wc, locale->info.lt)) + result |= PG_ISDIGIT; + if ((mask & PG_ISALPHA) && isalpha_l((unsigned char) wc, locale->info.lt)) + result |= PG_ISALPHA; + if ((mask & PG_ISUPPER) && isupper_l((unsigned char) wc, locale->info.lt)) + result |= PG_ISUPPER; + if ((mask & PG_ISLOWER) && islower_l((unsigned char) wc, locale->info.lt)) + result |= PG_ISLOWER; + if ((mask & PG_ISGRAPH) && isgraph_l((unsigned char) wc, locale->info.lt)) + result |= PG_ISGRAPH; + if ((mask & PG_ISPRINT) && isprint_l((unsigned char) wc, locale->info.lt)) + result |= PG_ISPRINT; + if ((mask & PG_ISPUNCT) && ispunct_l((unsigned char) wc, locale->info.lt)) + result |= PG_ISPUNCT; + if ((mask & PG_ISSPACE) && isspace_l((unsigned char) wc, locale->info.lt)) + result |= PG_ISSPACE; + + return result; +} + +static int +char_properties_libc_wide(pg_wchar wc, int mask, pg_locale_t locale) +{ + int result = 0; + + Assert(!locale->ctype_is_c); + Assert(GetDatabaseEncoding() == PG_UTF8); + + /* if wchar_t cannot represent the value, just return 0 */ + if (sizeof(wchar_t) < 4 && wc > (pg_wchar) 0xFFFF) + return 0; + + if ((mask & PG_ISDIGIT) && iswdigit_l((wint_t) wc, locale->info.lt)) + result |= PG_ISDIGIT; + if ((mask & PG_ISALPHA) && iswalpha_l((wint_t) wc, locale->info.lt)) + result |= PG_ISALPHA; + if ((mask & PG_ISUPPER) && iswupper_l((wint_t) wc, locale->info.lt)) + result |= PG_ISUPPER; + if ((mask & PG_ISLOWER) && iswlower_l((wint_t) wc, locale->info.lt)) + result |= PG_ISLOWER; + if ((mask & PG_ISGRAPH) && iswgraph_l((wint_t) wc, locale->info.lt)) + result |= PG_ISGRAPH; + if ((mask & PG_ISPRINT) && iswprint_l((wint_t) wc, locale->info.lt)) + result |= PG_ISPRINT; + if ((mask & PG_ISPUNCT) && iswpunct_l((wint_t) wc, locale->info.lt)) + result |= PG_ISPUNCT; + if ((mask & PG_ISSPACE) && iswspace_l((wint_t) wc, locale->info.lt)) + result |= PG_ISSPACE; + + return result; +} + +static pg_wchar +toupper_libc_1byte(pg_wchar wc, pg_locale_t locale) +{ + Assert(GetDatabaseEncoding() != PG_UTF8); + + if (wc <= (pg_wchar) UCHAR_MAX) + return toupper_l((unsigned char) wc, locale->info.lt); + else + return wc; +} + +static pg_wchar +toupper_libc_wide(pg_wchar wc, pg_locale_t locale) +{ + Assert(GetDatabaseEncoding() == PG_UTF8); + + if (sizeof(wchar_t) >= 4 || wc <= (pg_wchar) 0xFFFF) + return towupper_l((wint_t) wc, locale->info.lt); + else + return wc; +} + +static pg_wchar +tolower_libc_1byte(pg_wchar wc, pg_locale_t locale) +{ + Assert(GetDatabaseEncoding() != PG_UTF8); + + if (wc <= (pg_wchar) UCHAR_MAX) + return tolower_l((unsigned char) wc, locale->info.lt); + else + return wc; +} + +static pg_wchar +tolower_libc_wide(pg_wchar wc, pg_locale_t locale) +{ + Assert(GetDatabaseEncoding() == PG_UTF8); + + if (sizeof(wchar_t) >= 4 || wc <= (pg_wchar) 0xFFFF) + return towlower_l((wint_t) wc, locale->info.lt); + else + return wc; +} + /* * POSIX doesn't define _l-variants of these functions, but several systems * have them. We provide our own replacements here. diff --git a/src/include/utils/pg_locale.h b/src/include/utils/pg_locale.h index bbc10e0c3d..a5abf48bff 100644 --- a/src/include/utils/pg_locale.h +++ b/src/include/utils/pg_locale.h @@ -12,10 +12,25 @@ #ifndef _PG_LOCALE_ #define _PG_LOCALE_ +#include "mb/pg_wchar.h" + #ifdef USE_ICU #include #endif +/* + * Character properties for regular expressions. + */ +#define PG_ISDIGIT 0x01 +#define PG_ISALPHA 0x02 +#define PG_ISALNUM (PG_ISDIGIT | PG_ISALPHA) +#define PG_ISUPPER 0x04 +#define PG_ISLOWER 0x08 +#define PG_ISGRAPH 0x10 +#define PG_ISPRINT 0x20 +#define PG_ISPUNCT 0x40 +#define PG_ISSPACE 0x80 + #ifdef USE_ICU /* * ucol_strcollUTF8() was introduced in ICU 50, but it is buggy before ICU 53. @@ -104,6 +119,32 @@ struct casemap_methods pg_locale_t locale); }; +struct ctype_methods +{ + /* required */ + int (*char_properties) (pg_wchar wc, int mask, pg_locale_t locale); + + /* required */ + bool (*char_is_cased) (char ch, pg_locale_t locale); + + /* + * Optional. If defined, will only be called for single-byte encodings. If + * not defined, or if the encoding is multibyte, will fall back to + * pg_strlower(). + */ + char (*char_tolower) (unsigned char ch, pg_locale_t locale); + + /* required */ + pg_wchar (*wc_toupper) (pg_wchar wc, pg_locale_t locale); + pg_wchar (*wc_tolower) (pg_wchar wc, pg_locale_t locale); + + /* + * For regex and pattern matching efficiency, the maximum char value + * supported by the above methods. If zero, limit is set by regex code. + */ + pg_wchar max_chr; +}; + /* * We use a discriminated union to hold either a locale_t or an ICU collator. * pg_locale_t is occasionally checked for truth, so make it a pointer. @@ -129,6 +170,7 @@ struct pg_locale_struct const struct collate_methods *collate; /* NULL if collate_is_c */ const struct casemap_methods *casemap; /* NULL if ctype_is_c */ + const struct ctype_methods *ctype; /* NULL if ctype_is_c */ union { @@ -153,6 +195,10 @@ extern void init_database_collation(void); extern pg_locale_t pg_newlocale_from_collation(Oid collid); extern char *get_collation_actual_version(char collprovider, const char *collcollate); +extern int char_properties(pg_wchar wc, int mask, pg_locale_t locale); +extern bool char_is_cased(char ch, pg_locale_t locale); +extern bool char_tolower_enabled(pg_locale_t locale); +extern char char_tolower(unsigned char ch, pg_locale_t locale); extern size_t pg_strlower(char *dest, size_t destsize, const char *src, ssize_t srclen, pg_locale_t locale); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 57de1acff3..bbc1ac179e 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -1821,7 +1821,6 @@ PGTargetServerType PGTernaryBool PGTransactionStatusType PGVerbosity -PG_Locale_Strategy PG_Lock_Status PG_init_t PGcancel -- 2.45.2