From e717bf68aac6de98cd03b3f059ba90a360ce1ed9 Mon Sep 17 00:00:00 2001 From: shihao zhong Date: Sat, 24 Oct 2026 01:15:52 -0400 Subject: [PATCH v1 1/2] Reject unassigned bytes in single-byte encodings Bytes with no character in WIN1252 and twelve other encodings were accepted and then failed to convert. Discussion: https://postgr.es/m/2979839.1792127496@sss.pgh.pa.us --- src/backend/utils/adt/oracle_compat.c | 7 + .../latin2_and_win1250/latin2_and_win1250.c | 8 +- src/backend/utils/mb/mbutils.c | 9 +- src/common/wchar.c | 168 ++++++++++++++++-- 4 files changed, 171 insertions(+), 21 deletions(-) diff --git a/src/backend/utils/adt/oracle_compat.c b/src/backend/utils/adt/oracle_compat.c index 7422a454397..fd2455ad590 100644 --- a/src/backend/utils/adt/oracle_compat.c +++ b/src/backend/utils/adt/oracle_compat.c @@ -1138,6 +1138,13 @@ chr (PG_FUNCTION_ARGS) result = (text *) palloc(VARHDRSZ + 1); SET_VARSIZE(result, VARHDRSZ + 1); *VARDATA(result) = (char) cvalue; + + /* Some single-byte encodings leave byte values unassigned */ + if (!pg_verifymbstr(VARDATA(result), 1, true)) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("requested character not valid for encoding: %u", + cvalue))); } PG_RETURN_TEXT_P(result); diff --git a/src/backend/utils/mb/conversion_procs/latin2_and_win1250/latin2_and_win1250.c b/src/backend/utils/mb/conversion_procs/latin2_and_win1250/latin2_and_win1250.c index 29201c94434..6e5ca99c613 100644 --- a/src/backend/utils/mb/conversion_procs/latin2_and_win1250/latin2_and_win1250.c +++ b/src/backend/utils/mb/conversion_procs/latin2_and_win1250/latin2_and_win1250.c @@ -59,10 +59,10 @@ static const unsigned char win1250_2_iso88592[] = { /* ISO-8859-2 to WIN1250 */ static const unsigned char iso88592_2_win1250[] = { - 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, - 0x88, 0x89, 0x00, 0x8B, 0x00, 0x00, 0x00, 0x00, - 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, - 0x98, 0x99, 0x00, 0x9B, 0x00, 0x00, 0x00, 0x00, + 0x80, 0x00, 0x82, 0x00, 0x84, 0x85, 0x86, 0x87, + 0x00, 0x89, 0x00, 0x8B, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x00, 0x99, 0x00, 0x9B, 0x00, 0x00, 0x00, 0x00, 0xA0, 0xA5, 0xA2, 0xA3, 0xA4, 0xBC, 0x8C, 0xA7, 0xA8, 0x8A, 0xAA, 0x8D, 0x8F, 0xAD, 0x8E, 0xAF, 0xB0, 0xB9, 0xB2, 0xB3, 0xB4, 0xBE, 0x9C, 0xA1, diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c index b57dba4ead1..1e4a3fc5a3c 100644 --- a/src/backend/utils/mb/mbutils.c +++ b/src/backend/utils/mb/mbutils.c @@ -1729,17 +1729,18 @@ pg_verify_mbstr_len(int encoding, const char *mbstr, int len, bool noError) Assert(PG_VALID_ENCODING(encoding)); /* - * In single-byte encodings, we need only reject nulls (\0). + * In single-byte encodings, the string length is the character count. */ if (pg_encoding_max_length(encoding) <= 1) { - const char *nullpos = memchr(mbstr, 0, len); + int oklen; - if (nullpos == NULL) + oklen = pg_wchar_table[encoding].mbverifystr((const unsigned char *) mbstr, len); + if (oklen == len) return len; if (noError) return -1; - report_invalid_encoding(encoding, nullpos, 1); + report_invalid_encoding(encoding, mbstr + oklen, 1); } /* fetch function pointer just once */ diff --git a/src/common/wchar.c b/src/common/wchar.c index 926823cabec..65b2632eeb0 100644 --- a/src/common/wchar.c +++ b/src/common/wchar.c @@ -1224,6 +1224,148 @@ pg_latin1_verifystr(const unsigned char *s, int len) return nullpos - s; } +/* + * Some single-byte encodings leave byte values unassigned. We reject those + * as well as NUL, so that the database can't hold bytes that have no + * character and so can't be converted to any other encoding. Each bitmap + * covers the bytes 0x80 to 0xFF, and has a bit set for each byte with no + * entry in the encoding's *_to_utf8.map. The comments list those bytes. + */ + +/* + * A1 A2 A3 A5 A6 A7 A8 A9 AA AB AE AF B0 B1 B2 B3 B4 B5 B6 B7 B8 B9 + * BA BC BD BE C0 DB DC DD DE DF F3 F4 F5 F6 F7 F8 F9 FA FB FC FD FE + * FF + */ +static const uint64 iso_8859_6_unassigned[2] = { + UINT64CONST(0x77FFCFEE00000000), UINT64CONST(0xFFF80000F8000001) +}; + +/* AE D2 FF */ +static const uint64 iso_8859_7_unassigned[2] = { + UINT64CONST(0x0000400000000000), UINT64CONST(0x8000000000040000) +}; + +/* + * A1 BF C0 C1 C2 C3 C4 C5 C6 C7 C8 C9 CA CB CC CD CE CF D0 D1 D2 D3 + * D4 D5 D6 D7 D8 D9 DA DB DC DD DE FB FC FF + */ +static const uint64 iso_8859_8_unassigned[2] = { + UINT64CONST(0x8000000200000000), UINT64CONST(0x980000007FFFFFFF) +}; + +/* A5 AE BE C3 D0 E3 F0 */ +static const uint64 latin3_unassigned[2] = { + UINT64CONST(0x4000402000000000), UINT64CONST(0x0001000800010008) +}; + +/* 81 83 88 90 98 */ +static const uint64 win1250_unassigned[2] = { + UINT64CONST(0x000000000101010A), UINT64CONST(0x0000000000000000) +}; + +/* 98 */ +static const uint64 win1251_unassigned[2] = { + UINT64CONST(0x0000000001000000), UINT64CONST(0x0000000000000000) +}; + +/* 81 8D 8F 90 9D */ +static const uint64 win1252_unassigned[2] = { + UINT64CONST(0x000000002001A002), UINT64CONST(0x0000000000000000) +}; + +/* 81 88 8A 8C 8D 8E 8F 90 98 9A 9C 9D 9E 9F AA D2 FF */ +static const uint64 win1253_unassigned[2] = { + UINT64CONST(0x00000400F501F502), UINT64CONST(0x8000000000040000) +}; + +/* 81 8D 8E 8F 90 9D 9E */ +static const uint64 win1254_unassigned[2] = { + UINT64CONST(0x000000006001E002), UINT64CONST(0x0000000000000000) +}; + +/* + * 81 8A 8C 8D 8E 8F 90 9A 9C 9D 9E 9F CA D9 DA DB DC DD DE DF FB FC + * FF + */ +static const uint64 win1255_unassigned[2] = { + UINT64CONST(0x00000000F401F402), UINT64CONST(0x98000000FE000400) +}; + +/* 81 83 88 8A 8C 90 98 9A 9C 9F A1 A5 */ +static const uint64 win1257_unassigned[2] = { + UINT64CONST(0x000000229501150A), UINT64CONST(0x0000000000000000) +}; + +/* 81 8A 8D 8E 8F 90 9A 9D 9E */ +static const uint64 win1258_unassigned[2] = { + UINT64CONST(0x000000006401E402), UINT64CONST(0x0000000000000000) +}; + +/* + * 81 82 83 84 86 87 88 89 8A 8B 8C 8D 8E 8F 90 98 99 9A 9B 9C 9D 9E + * 9F DB DC DD DE FC FD FE FF + */ +static const uint64 win874_unassigned[2] = { + UINT64CONST(0x00000000FF01FFDE), UINT64CONST(0xF000000078000000) +}; + +static inline bool +singlebyte_is_valid(unsigned char c, const uint64 *unassigned) +{ + /* No branches here, as text often mixes ASCII and high-bit bytes */ + uint64 hole = (unassigned[(c >> 6) & 1] >> (c & 63)) & (c >> 7); + + return (c != '\0') & ((hole & 1) == 0); +} + +static int +pg_singlebyte_verifystr(const unsigned char *s, int len, + const uint64 *unassigned) +{ + const unsigned char *start = s; + + while (len > 0) + { + int n = Min(len, (int) sizeof(Vector8)); + + /* Skip a chunk of plain ASCII quickly, else check it byte by byte */ + if (n < (int) sizeof(Vector8) || !is_valid_ascii(s, n)) + { + for (int i = 0; i < n; i++) + { + if (!singlebyte_is_valid(s[i], unassigned)) + return s + i - start; + } + } + s += n; + len -= n; + } + + return s - start; +} + +#define SINGLEBYTE_VERIFYSTR(enc) \ +static int \ +pg_##enc##_verifystr(const unsigned char *s, int len) \ +{ \ + return pg_singlebyte_verifystr(s, len, enc##_unassigned); \ +} + +SINGLEBYTE_VERIFYSTR(iso_8859_6) +SINGLEBYTE_VERIFYSTR(iso_8859_7) +SINGLEBYTE_VERIFYSTR(iso_8859_8) +SINGLEBYTE_VERIFYSTR(latin3) +SINGLEBYTE_VERIFYSTR(win1250) +SINGLEBYTE_VERIFYSTR(win1251) +SINGLEBYTE_VERIFYSTR(win1252) +SINGLEBYTE_VERIFYSTR(win1253) +SINGLEBYTE_VERIFYSTR(win1254) +SINGLEBYTE_VERIFYSTR(win1255) +SINGLEBYTE_VERIFYSTR(win1257) +SINGLEBYTE_VERIFYSTR(win1258) +SINGLEBYTE_VERIFYSTR(win874) + static int pg_sjis_verifychar(const unsigned char *s, int len) { @@ -1872,7 +2014,7 @@ const pg_wchar_tbl pg_wchar_table[] = { [PG_UTF8] = {pg_utf2wchar_with_len, pg_wchar2utf_with_len, pg_utf_mblen, pg_utf_dsplen, pg_utf8_verifychar, pg_utf8_verifystr, 4}, [PG_LATIN1] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1}, [PG_LATIN2] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1}, - [PG_LATIN3] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1}, + [PG_LATIN3] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin3_verifystr, 1}, [PG_LATIN4] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1}, [PG_LATIN5] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1}, [PG_LATIN6] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1}, @@ -1881,21 +2023,21 @@ const pg_wchar_tbl pg_wchar_table[] = { [PG_LATIN9] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1}, [PG_LATIN10] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1}, [PG_WIN1256] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1}, - [PG_WIN1258] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1}, + [PG_WIN1258] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_win1258_verifystr, 1}, [PG_WIN866] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1}, - [PG_WIN874] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1}, + [PG_WIN874] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_win874_verifystr, 1}, [PG_KOI8R] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1}, - [PG_WIN1251] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1}, - [PG_WIN1252] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1}, + [PG_WIN1251] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_win1251_verifystr, 1}, + [PG_WIN1252] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_win1252_verifystr, 1}, [PG_ISO_8859_5] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1}, - [PG_ISO_8859_6] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1}, - [PG_ISO_8859_7] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1}, - [PG_ISO_8859_8] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1}, - [PG_WIN1250] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1}, - [PG_WIN1253] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1}, - [PG_WIN1254] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1}, - [PG_WIN1255] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1}, - [PG_WIN1257] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1}, + [PG_ISO_8859_6] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_iso_8859_6_verifystr, 1}, + [PG_ISO_8859_7] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_iso_8859_7_verifystr, 1}, + [PG_ISO_8859_8] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_iso_8859_8_verifystr, 1}, + [PG_WIN1250] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_win1250_verifystr, 1}, + [PG_WIN1253] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_win1253_verifystr, 1}, + [PG_WIN1254] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_win1254_verifystr, 1}, + [PG_WIN1255] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_win1255_verifystr, 1}, + [PG_WIN1257] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_win1257_verifystr, 1}, [PG_KOI8U] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1}, [PG_SJIS] = {0, 0, pg_sjis_mblen, pg_sjis_dsplen, pg_sjis_verifychar, pg_sjis_verifystr, 2}, [PG_BIG5] = {0, 0, pg_big5_mblen, pg_big5_dsplen, pg_big5_verifychar, pg_big5_verifystr, 2}, -- 2.37.1 (Apple Git-137.1)