From b09c5bc0a437fe196940d7633fac4985e5538a06 Mon Sep 17 00:00:00 2001 From: Jeff Davis Date: Mon, 13 Jul 2026 19:31:56 -0700 Subject: [PATCH v2] Generate unicode_limits.h. To ensure we do not overflow a size_t while case mapping on a 32-bit platform, we need to know the maximum amount a UTF8 string can expand. Calculate that maximum while generating unicode tables as a part of the update-unicode target, and output into a new header unicode_limits.h. Minor refactoring along the way. Add a StaticAssertDecl to check that a MaxAllocSize text value expanded by that amount would still have a length that fits in size_t. (We couldn't actually create a new text value out of that, but we still need to avoid overflow.) Reported-by: Tom Lane Discussion: https://postgr.es/m/3213927.1783950167@sss.pgh.pa.us --- src/backend/utils/adt/pg_locale_builtin.c | 11 + src/common/unicode/Makefile | 6 +- .../unicode/generate-unicode_case_table.pl | 194 ++++++++++++------ src/common/unicode/meson.build | 2 +- src/common/unicode_case.c | 12 +- src/include/common/unicode_case_table.h | 9 +- src/include/common/unicode_limits.h | 39 ++++ 7 files changed, 202 insertions(+), 71 deletions(-) create mode 100644 src/include/common/unicode_limits.h diff --git a/src/backend/utils/adt/pg_locale_builtin.c b/src/backend/utils/adt/pg_locale_builtin.c index 5619daf43c3..5c67c6ffc3a 100644 --- a/src/backend/utils/adt/pg_locale_builtin.c +++ b/src/backend/utils/adt/pg_locale_builtin.c @@ -15,11 +15,22 @@ #include "catalog/pg_collation.h" #include "common/unicode_case.h" #include "common/unicode_category.h" +#include "common/unicode_limits.h" #include "miscadmin.h" #include "utils/builtins.h" +#include "utils/memutils.h" #include "utils/pg_locale.h" #include "utils/syscache.h" +/* + * The largest text value must fit in MaxAllocSize, but then may grow during + * case mapping. While the resulting string will not be representable as a new + * text value, we must at least be sure not to overflow a size_t while + * processing it. + */ +StaticAssertDecl(SIZE_MAX / UTF8_MAX_CASEMAP_EXPANSION > MaxAllocSize, + "case mapping may overflow size_t"); + extern pg_locale_t create_pg_locale_builtin(Oid collid, MemoryContext context); extern char *get_collation_actual_version_builtin(const char *collcollate); diff --git a/src/common/unicode/Makefile b/src/common/unicode/Makefile index c150e403cda..714ccbe00bd 100644 --- a/src/common/unicode/Makefile +++ b/src/common/unicode/Makefile @@ -21,7 +21,7 @@ CPPFLAGS += $(ICU_CFLAGS) # By default, do nothing. all: -update-unicode: unicode_case_table.h unicode_category_table.h unicode_east_asian_fw_table.h unicode_nonspacing_table.h unicode_norm_hashfunc.h unicode_norm_table.h unicode_normprops_table.h unicode_version.h +update-unicode: unicode_case_table.h unicode_category_table.h unicode_east_asian_fw_table.h unicode_nonspacing_table.h unicode_norm_hashfunc.h unicode_limits.h unicode_norm_table.h unicode_normprops_table.h unicode_version.h mv $^ $(top_srcdir)/src/include/common/ $(MAKE) case-check $(MAKE) category-check @@ -36,6 +36,8 @@ CompositionExclusions.txt CaseFolding.txt DerivedCoreProperties.txt DerivedNorma unicode_version.h: generate-unicode_version.pl $(PERL) $< --version $(UNICODE_VERSION) +unicode_limits.h: unicode_case_table.h + unicode_case_table.h: generate-unicode_case_table.pl CaseFolding.txt SpecialCasing.txt UnicodeData.txt $(PERL) $< @@ -91,4 +93,4 @@ clean: rm -f $(OBJS) case_test case_test.o category_test category_test.o norm_test norm_test.o distclean: clean - rm -f CompositionExclusions.txt CaseFolding.txt DerivedCoreProperties.txt DerivedNormalizationProps.txt EastAsianWidth.txt NormalizationTest.txt PropList.txt SpecialCasing.txt UnicodeData.txt norm_test_table.h unicode_case_table.h unicode_category_table.h unicode_norm_table.h + rm -f CompositionExclusions.txt CaseFolding.txt DerivedCoreProperties.txt DerivedNormalizationProps.txt EastAsianWidth.txt NormalizationTest.txt PropList.txt SpecialCasing.txt UnicodeData.txt norm_test_table.h unicode_case_table.h unicode_category_table.h unicode_limits.h unicode_norm_table.h diff --git a/src/common/unicode/generate-unicode_case_table.pl b/src/common/unicode/generate-unicode_case_table.pl index c1b94f47f56..d1dd3a5e8f2 100644 --- a/src/common/unicode/generate-unicode_case_table.pl +++ b/src/common/unicode/generate-unicode_case_table.pl @@ -4,13 +4,14 @@ # or locale-specific mappings. # # Input: CaseFolding.txt SpecialCasing.txt UnicodeData.txt -# Output: unicode_case_table.h +# Output: unicode_case_table.h unicode_limits.h # # Copyright (c) 2000-2026, PostgreSQL Global Development Group use strict; use warnings FATAL => 'all'; use Getopt::Long; +use Encode qw(encode_utf8); use FindBin; use lib "$FindBin::RealBin/../../tools/"; @@ -19,11 +20,22 @@ my $output_path = '.'; GetOptions('outdir:s' => \$output_path); +my $output_limits_file = "$output_path/unicode_limits.h"; my $output_table_file = "$output_path/unicode_case_table.h"; +# The following are computed during table generation: + # The maximum number of codepoints that can result from case mapping -# of a single character. See Unicode section 5.18 "Case Mappings". -my $MAX_CASE_EXPANSION = 3; +# of a single character. +my $UNICODE_MAX_CASEMAP_CODEPOINTS = 0; + +# The maximum expansion factor (in bytes) when case mapping a UTF8 +# string. +my $UTF8_MAX_CASEMAP_EXPANSION = 0; + +# The maximum number of UTF8 bytes that can result from case mapping +# a single code point. +my $UTF8_CASEMAP_BUFSZ = 0; my $FH; @@ -44,15 +56,22 @@ while (my $line = <$FH>) die "Simple_Titlecase $code out of range" if $simple_titlecase > 0x10FFFF; die "Simple_Uppercase $code out of range" if $simple_uppercase > 0x10FFFF; - if ($simple_lowercase || $simple_titlecase || $simple_uppercase) - { - $simple{$code} = { - Simple_Lowercase => ($simple_lowercase || $code), - Simple_Titlecase => ($simple_titlecase || $code), - Simple_Uppercase => ($simple_uppercase || $code), - Simple_Foldcase => $code, - }; - } + next if !$simple_lowercase && !$simple_titlecase && !$simple_uppercase; + + $simple_lowercase ||= $code; + $simple_titlecase ||= $code; + $simple_uppercase ||= $code; + + update_limits('uppercase', $code, ($simple_uppercase)); + update_limits('lowercase', $code, ($simple_lowercase)); + update_limits('titlecase', $code, ($simple_titlecase)); + + $simple{$code} = { + Simple_Lowercase => $simple_lowercase, + Simple_Titlecase => $simple_titlecase, + Simple_Uppercase => $simple_uppercase, + Simple_Foldcase => $code, + }; } close $FH; @@ -105,19 +124,9 @@ while (my $line = <$FH>) push @upper, $code if (scalar @upper == 0); push @fold, $code; - # none should map to more than 3 codepoints - die "lowercase expansion for 0x$elts[0] exceeds maximum: '$elts[1]'" - if (scalar @lower) > $MAX_CASE_EXPANSION; - die "titlecase expansion for 0x$elts[0] exceeds maximum: '$elts[2]'" - if (scalar @title) > $MAX_CASE_EXPANSION; - die "uppercase expansion for 0x$elts[0] exceeds maximum: '$elts[3]'" - if (scalar @upper) > $MAX_CASE_EXPANSION; - - # pad arrays to a fixed length of 3 - while (scalar @upper < $MAX_CASE_EXPANSION) { push @upper, 0x000000 } - while (scalar @lower < $MAX_CASE_EXPANSION) { push @lower, 0x000000 } - while (scalar @title < $MAX_CASE_EXPANSION) { push @title, 0x000000 } - while (scalar @fold < $MAX_CASE_EXPANSION) { push @fold, 0x000000 } + update_limits('lowercase', $code, @lower); + update_limits('titlecase', $code, @title); + update_limits('uppercase', $code, @upper); # Characters with special mappings may not have simple mappings; # ensure that an entry exists. @@ -174,6 +183,8 @@ while (my $line = <$FH>) die "unsupported status type '$status'" if $status ne 'S' && $status ne 'C' && $status ne 'F'; + update_limits('casefold', $code, @fold); + # initialize simple case mappings if they don't exist $simple{$code} ||= { Simple_Lowercase => $code, @@ -197,26 +208,12 @@ while (my $line = <$FH>) if ($status eq 'F' || ($status eq 'C' && defined $special{$code})) { - while (scalar @fold < $MAX_CASE_EXPANSION) { push @fold, 0x000000 } - #initialize special case mappings if they don't exist if (!defined $special{$code}) { my @lower = ($simple{$code}{Simple_Lowercase}); my @title = ($simple{$code}{Simple_Titlecase}); my @upper = ($simple{$code}{Simple_Uppercase}); - while (scalar @lower < $MAX_CASE_EXPANSION) - { - push @lower, 0x000000; - } - while (scalar @title < $MAX_CASE_EXPANSION) - { - push @title, 0x000000; - } - while (scalar @upper < $MAX_CASE_EXPANSION) - { - push @upper, 0x000000; - } $special{$code} = { Lowercase => \@lower, Titlecase => \@title, @@ -245,6 +242,53 @@ die "special case map contains $num_special entries which cannot be represented in uint8" if ($num_special > 256); +open my $LIMITS, '>', $output_limits_file + or die "Could not open output file $output_limits_file: $!\n"; + +print $LIMITS <<"EOS"; +/*------------------------------------------------------------------------- + * + * unicode_limits.h + * Useful limits calculated from Unicode data. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/common/unicode_limits.h + * + *------------------------------------------------------------------------- + */ + +#ifndef UNICODE_LIMITS_H +#define UNICODE_LIMITS_H + +/* + * File auto-generated by src/common/unicode/generate-unicode_case_table.pl, + * do not edit. + */ + +/* + * The maximum number of codepoints that can result from case mapping + * of a single code point. See Unicode section 5.18 "Case Mappings". + */ +#define UNICODE_MAX_CASEMAP_CODEPOINTS $UNICODE_MAX_CASEMAP_CODEPOINTS + +/* + * The maximum expansion factor of a UTF8 string due to case mapping. + */ +#define UTF8_MAX_CASEMAP_EXPANSION $UTF8_MAX_CASEMAP_EXPANSION + +/* + * The maximum number of UTF8 bytes needed to store the result of case + * mapping a single code point. + */ +#define UTF8_CASEMAP_BUFSZ $UTF8_CASEMAP_BUFSZ + +#endif +EOS + +close $LIMITS; + # Start writing out the output files open my $OT, '>', $output_table_file or die "Could not open output file $output_table_file: $!\n"; @@ -270,12 +314,7 @@ print $OT <<"EOS"; */ #include "common/unicode_case.h" - -/* - * The maximum number of codepoints that can result from case mapping - * of a single character. See Unicode section 5.18 "Case Mappings". - */ -#define MAX_CASE_EXPANSION 3 +#include "common/unicode_limits.h" /* * Case mapping condition flags. For now, only Final_Sigma is supported. @@ -296,7 +335,7 @@ typedef enum typedef struct { int16 conditions; - char32_t map[NCaseKind][MAX_CASE_EXPANSION]; + char32_t map[NCaseKind][UNICODE_MAX_CASEMAP_CODEPOINTS]; } pg_special_case; /* @@ -310,18 +349,26 @@ EOS foreach my $code (sort { $a <=> $b } (keys %special)) { - die if scalar @{ $special{$code}{Lowercase} } != $MAX_CASE_EXPANSION; - die if scalar @{ $special{$code}{Titlecase} } != $MAX_CASE_EXPANSION; - die if scalar @{ $special{$code}{Uppercase} } != $MAX_CASE_EXPANSION; - die if scalar @{ $special{$code}{Foldcase} } != $MAX_CASE_EXPANSION; - my $lower = join ", ", - (map { sprintf "0x%06x", $_ } @{ $special{$code}{Lowercase} }); - my $title = join ", ", - (map { sprintf "0x%06x", $_ } @{ $special{$code}{Titlecase} }); - my $upper = join ", ", - (map { sprintf "0x%06x", $_ } @{ $special{$code}{Uppercase} }); - my $fold = join ", ", - (map { sprintf "0x%06x", $_ } @{ $special{$code}{Foldcase} }); + my @lower = @{ $special{$code}{Lowercase} }; + my @title = @{ $special{$code}{Titlecase} }; + my @upper = @{ $special{$code}{Uppercase} }; + my @fold = @{ $special{$code}{Foldcase} }; + + # pad to fixed length + push @lower, 0x000000 + while (scalar @lower < $UNICODE_MAX_CASEMAP_CODEPOINTS); + push @title, 0x000000 + while (scalar @title < $UNICODE_MAX_CASEMAP_CODEPOINTS); + push @upper, 0x000000 + while (scalar @upper < $UNICODE_MAX_CASEMAP_CODEPOINTS); + push @fold, 0x000000 + while (scalar @fold < $UNICODE_MAX_CASEMAP_CODEPOINTS); + + my $lower = join ", ", (map { sprintf "0x%06x", $_ } @lower); + my $title = join ", ", (map { sprintf "0x%06x", $_ } @title); + my $upper = join ", ", (map { sprintf "0x%06x", $_ } @upper); + my $fold = join ", ", (map { sprintf "0x%06x", $_ } @fold); + printf $OT "\t{%s, ", $special{$code}{Conditions}; printf $OT "{[CaseLower] = {%s},[CaseTitle] = {%s},[CaseUpper] = {%s},[CaseFold] = {%s}}},\n", @@ -522,6 +569,37 @@ EOS close $OT; +# calculate, check, and update limits +sub update_limits +{ + my ($casekind, $codepoint, @mapping) = @_; + my $n = scalar @mapping; + + # See Unicode section 5.18 "Case Mappings" + my $mapping_str = join ' ', map { sprintf "0x%06x", $_ } @mapping; + die sprintf("%s expansion for 0x%06x exceeds maximum: %s", + $casekind, $codepoint, $mapping_str) + if $n > 3; + + $UNICODE_MAX_CASEMAP_CODEPOINTS = $n + if $n > $UNICODE_MAX_CASEMAP_CODEPOINTS; + + my $utf8len1 = length(encode_utf8(chr $codepoint)); + my $utf8len2 = length(encode_utf8(join '', map { chr } @mapping)); + my $factor = int(($utf8len1 + $utf8len2 - 1) / $utf8len1); + $UTF8_MAX_CASEMAP_EXPANSION = $factor + if $factor > $UTF8_MAX_CASEMAP_EXPANSION; + + # Not guaranteed by Unicode standard. If larger than expected, + # then evaluate C type overflow risks before using the generated + # table. + die "UTF8 casemap expansion is greater than 3" + if $UTF8_MAX_CASEMAP_EXPANSION > 3; + + $UTF8_CASEMAP_BUFSZ = $utf8len2 + if $utf8len2 > $UTF8_CASEMAP_BUFSZ; +} + # The function generates C code with a series of nested if-else conditions # to search for the matching interval. sub branch diff --git a/src/common/unicode/meson.build b/src/common/unicode/meson.build index 8a31fe40f6c..a32d44d1816 100644 --- a/src/common/unicode/meson.build +++ b/src/common/unicode/meson.build @@ -28,7 +28,7 @@ update_unicode_common_targets = [] update_unicode_common_targets += \ custom_target('unicode_case_table.h', input: [unicode_data['CaseFolding.txt'], unicode_data['SpecialCasing.txt'], unicode_data['UnicodeData.txt']], - output: ['unicode_case_table.h'], + output: ['unicode_case_table.h', 'unicode_limits.h'], command: [ perl, files('generate-unicode_case_table.pl'), '--outdir', '@OUTDIR@', '@INPUT@'], diff --git a/src/common/unicode_case.c b/src/common/unicode_case.c index 399f08235a9..00668a7d4f3 100644 --- a/src/common/unicode_case.c +++ b/src/common/unicode_case.c @@ -237,6 +237,12 @@ convert_case(char *dst, size_t dstsize, const char *src, size_t srclen, size_t result_len = 0; size_t boundary = 0; + /* + * Must be guaranteed by caller to avoid overflow (text values limited to + * MaxAllocSize anyway). + */ + Assert(srclen < SIZE_MAX / UTF8_MAX_CASEMAP_EXPANSION); + Assert((str_casekind == CaseTitle && wbnext && wbstate) || (str_casekind != CaseTitle && !wbnext && !wbstate)); @@ -299,9 +305,9 @@ convert_case(char *dst, size_t dstsize, const char *src, size_t srclen, } break; case CASEMAP_SPECIAL: - /* replace with up to MAX_CASE_EXPANSION characters */ + /* replace with up to UNICODE_MAX_CASEMAP_CODEPOINTS */ Assert(simple == 0); - for (int i = 0; i < MAX_CASE_EXPANSION && special[i]; i++) + for (int i = 0; i < UNICODE_MAX_CASEMAP_CODEPOINTS && special[i]; i++) { char32_t u2 = special[i]; size_t u2len = unicode_utf8len(u2); @@ -415,7 +421,7 @@ check_special_conditions(int conditions, const char *str, size_t len, * * If full is true, and a special case mapping is found and the conditions are * met, 'special' is set to the mapping result (which is an array of up to - * MAX_CASE_EXPANSION characters) and CASEMAP_SPECIAL is returned. + * UNICODE_MAX_CASEMAP_CODEPOINTS) and CASEMAP_SPECIAL is returned. * * Otherwise, search for a simple mapping, and if found, set 'simple' to the * result and return CASEMAP_SIMPLE. diff --git a/src/include/common/unicode_case_table.h b/src/include/common/unicode_case_table.h index b2274627c4f..9930943453f 100644 --- a/src/include/common/unicode_case_table.h +++ b/src/include/common/unicode_case_table.h @@ -18,12 +18,7 @@ */ #include "common/unicode_case.h" - -/* - * The maximum number of codepoints that can result from case mapping - * of a single character. See Unicode section 5.18 "Case Mappings". - */ -#define MAX_CASE_EXPANSION 3 +#include "common/unicode_limits.h" /* * Case mapping condition flags. For now, only Final_Sigma is supported. @@ -44,7 +39,7 @@ typedef enum typedef struct { int16 conditions; - char32_t map[NCaseKind][MAX_CASE_EXPANSION]; + char32_t map[NCaseKind][UNICODE_MAX_CASEMAP_CODEPOINTS]; } pg_special_case; /* diff --git a/src/include/common/unicode_limits.h b/src/include/common/unicode_limits.h new file mode 100644 index 00000000000..59e307857ed --- /dev/null +++ b/src/include/common/unicode_limits.h @@ -0,0 +1,39 @@ +/*------------------------------------------------------------------------- + * + * unicode_limits.h + * Useful limits calculated from Unicode data. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/common/unicode_limits.h + * + *------------------------------------------------------------------------- + */ + +#ifndef UNICODE_LIMITS_H +#define UNICODE_LIMITS_H + +/* + * File auto-generated by src/common/unicode/generate-unicode_case_table.pl, + * do not edit. + */ + +/* + * The maximum number of codepoints that can result from case mapping + * of a single code point. See Unicode section 5.18 "Case Mappings". + */ +#define UNICODE_MAX_CASEMAP_CODEPOINTS 3 + +/* + * The maximum expansion factor of a UTF8 string due to case mapping. + */ +#define UTF8_MAX_CASEMAP_EXPANSION 3 + +/* + * The maximum number of UTF8 bytes needed to store the result of case + * mapping a single code point. + */ +#define UTF8_CASEMAP_BUFSZ 6 + +#endif -- 2.43.0