From b825cc902f2657028c68a2738c2852f3365c96cd Mon Sep 17 00:00:00 2001 From: shihao zhong Date: Sat, 24 Oct 2026 01:15:52 -0400 Subject: [PATCH v1 2/2] Add tests for unassigned bytes in single-byte encodings Check that WIN1252 rejects 0x81 but keeps 0x80, and that LATIN2 no longer converts into WIN1250 holes. Discussion: https://postgr.es/m/2979839.1792127496@sss.pgh.pa.us --- src/test/regress/expected/conversion.out | 47 ++++++++++++++++++++++++ src/test/regress/sql/conversion.sql | 21 +++++++++++ 2 files changed, 68 insertions(+) diff --git a/src/test/regress/expected/conversion.out b/src/test/regress/expected/conversion.out index 01491ac6338..63c831a9a3d 100644 --- a/src/test/regress/expected/conversion.out +++ b/src/test/regress/expected/conversion.out @@ -588,6 +588,53 @@ select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'koi8r')).* from i invalid, NUL byte | \xe4dede00 | \xc6cfcf | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) +-- +-- WIN1252 +-- +CREATE TABLE win1252_inputs (inbytes bytea, description text); +insert into win1252_inputs values + ('\x666f6f', 'valid, pure ASCII'), + ('\x80e9', 'valid'), + ('\x81', 'invalid, unassigned byte'), + ('\x80e99d', 'invalid, unassigned byte'), + ('\x00', 'invalid, NUL byte'); +-- Test WIN1252 verification +select description, inbytes, (test_conv(inbytes, 'win1252', 'win1252')).* from win1252_inputs; + description | inbytes | result | errorat | error +--------------------------+----------+----------+---------+---------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x80e9 | \x80e9 | | + invalid, unassigned byte | \x81 | \x | \x81 | invalid byte sequence for encoding "WIN1252": 0x81 + invalid, unassigned byte | \x80e99d | \x80e9 | \x9d | invalid byte sequence for encoding "WIN1252": 0x9d + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "WIN1252": 0x00 +(5 rows) + +select description, length(inbytes, 'win1252') from win1252_inputs where inbytes = '\x80e9'; + description | length +-------------+-------- + valid | 2 +(1 row) + +select description, length(inbytes, 'win1252') from win1252_inputs where inbytes = '\x80e99d'; +ERROR: invalid byte sequence for encoding "WIN1252": 0x9d +-- Test conversions from WIN1252 +select description, inbytes, (test_conv(inbytes, 'win1252', 'utf8')).* from win1252_inputs; + description | inbytes | result | errorat | error +--------------------------+----------+--------------+---------+---------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x80e9 | \xe282acc3a9 | | + invalid, unassigned byte | \x81 | \x | \x81 | character with byte sequence 0x81 in encoding "WIN1252" has no equivalent in encoding "UTF8" + invalid, unassigned byte | \x80e99d | \xe282acc3a9 | \x9d | character with byte sequence 0x9d in encoding "WIN1252" has no equivalent in encoding "UTF8" + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "WIN1252": 0x00 +(5 rows) + +-- LATIN2 control characters that WIN1250 has no byte for +select (test_conv('\x418142', 'latin2', 'win1250')).*; + result | errorat | error +--------+---------+------------------------------------------------------------------------------------------------ + \x41 | \x8142 | character with byte sequence 0x81 in encoding "LATIN2" has no equivalent in encoding "WIN1250" +(1 row) + -- -- Big5 -- diff --git a/src/test/regress/sql/conversion.sql b/src/test/regress/sql/conversion.sql index c9a76cb014f..fd9ed3edbba 100644 --- a/src/test/regress/sql/conversion.sql +++ b/src/test/regress/sql/conversion.sql @@ -330,6 +330,27 @@ select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'iso8859-5')).* fr select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'utf8')).* from iso8859_5_inputs; select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'koi8r')).* from iso8859_5_inputs; +-- +-- WIN1252 +-- +CREATE TABLE win1252_inputs (inbytes bytea, description text); +insert into win1252_inputs values + ('\x666f6f', 'valid, pure ASCII'), + ('\x80e9', 'valid'), + ('\x81', 'invalid, unassigned byte'), + ('\x80e99d', 'invalid, unassigned byte'), + ('\x00', 'invalid, NUL byte'); + +-- Test WIN1252 verification +select description, inbytes, (test_conv(inbytes, 'win1252', 'win1252')).* from win1252_inputs; +select description, length(inbytes, 'win1252') from win1252_inputs where inbytes = '\x80e9'; +select description, length(inbytes, 'win1252') from win1252_inputs where inbytes = '\x80e99d'; +-- Test conversions from WIN1252 +select description, inbytes, (test_conv(inbytes, 'win1252', 'utf8')).* from win1252_inputs; + +-- LATIN2 control characters that WIN1250 has no byte for +select (test_conv('\x418142', 'latin2', 'win1250')).*; + -- -- Big5 -- -- 2.37.1 (Apple Git-137.1)