From 3ebaaada53f1a04dc7e807b31ba54f420c967b42 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Tue, 27 Oct 2026 14:12:49 +0100 Subject: [PATCH] Clean up code comments for pg_wchar_tbl The documentation for the functions in the fields of pg_wchar_tbl was spread out all over the place and was missing a lot of detail. Move and expand this at the definition of the struct. (See also struct TableAmRoutine in access/tableam.h for inspiration.) Also remove unnecessary typedefs, except where they were used elsewhere as well. This indirection makes the code documentation harder to follow. And some of those typedefs had nonsensical names, like *_converter for functions that didn't do any converting. --- src/backend/utils/mb/mbutils.c | 10 +-- src/common/wchar.c | 60 +-------------- src/include/mb/pg_wchar.h | 125 +++++++++++++++++++++++++------ src/tools/pgindent/typedefs.list | 8 +- 4 files changed, 113 insertions(+), 90 deletions(-) diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c index b57dba4ead1..bdfdd9eeace 100644 --- a/src/backend/utils/mb/mbutils.c +++ b/src/backend/utils/mb/mbutils.c @@ -1222,7 +1222,7 @@ int pg_encoding_mbcliplen(int encoding, const char *mbstr, int len, int limit) { - mblen_converter mblen_fn; + mblen_function mblen_fn; int clen = 0; int l; @@ -1452,15 +1452,15 @@ static bool pg_generic_charinc(unsigned char *charptr, int len) { unsigned char *lastbyte = charptr + len - 1; - mbchar_verifier mbverify; + mbverifychar_function mbverifychar; /* We can just invoke the character verifier directly. */ - mbverify = pg_wchar_table[GetDatabaseEncoding()].mbverifychar; + mbverifychar = pg_wchar_table[GetDatabaseEncoding()].mbverifychar; while (*lastbyte < (unsigned char) 255) { (*lastbyte)++; - if ((*mbverify) (charptr, len) == len) + if ((*mbverifychar) (charptr, len) == len) return true; } @@ -1723,7 +1723,7 @@ pg_verify_mbstr(int encoding, const char *mbstr, int len, bool noError) int pg_verify_mbstr_len(int encoding, const char *mbstr, int len, bool noError) { - mbchar_verifier mbverifychar; + mbverifychar_function mbverifychar; int mb_len; Assert(PG_VALID_ENCODING(encoding)); diff --git a/src/common/wchar.c b/src/common/wchar.c index 926823cabec..23bbdbd2ae5 100644 --- a/src/common/wchar.c +++ b/src/common/wchar.c @@ -37,32 +37,6 @@ #define NONUTF8_INVALID_BYTE1 (' ') -/* - * Operations on multi-byte encodings are driven by a table of helper - * functions. - * - * To add an encoding support, define mblen(), dsplen(), verifychar() and - * verifystr() for the encoding. For server-encodings, also define mb2wchar() - * and wchar2mb() conversion functions. - * - * These functions generally assume that their input is validly formed. - * The "verifier" functions, further down in the file, have to be more - * paranoid. - * - * We expect that mblen() does not need to examine more than the first byte - * of the character to discover the correct length. GB18030 is an exception - * to that rule, though, as it also looks at second byte. But even that - * behaves in a predictable way, if you only pass the first byte: it will - * treat 4-byte encoded characters as two 2-byte encoded characters, which is - * good enough for all current uses. - * - * Note: for the display output of psql to work properly, the return values - * of the dsplen functions must conform to the Unicode standard. In particular - * the NUL character is zero width and control characters are generally - * width -1. It is recommended that non-ASCII encodings refer their ASCII - * subset to the ASCII routines to ensure consistency. - */ - /* No error-reporting facility. Ignore incomplete trailing byte sequence. */ #define MB2CHAR_NEED_AT_LEAST(len, need) if ((len) < (need)) break @@ -390,9 +364,6 @@ pg_euctw_dsplen(const unsigned char *s) /* * Convert pg_wchar to EUC_* encoding. - * caller must allocate enough space for "to", including a trailing zero! - * len: length of from. - * "from" not necessarily null terminated. */ static int pg_wchar2euc_with_len(const pg_wchar *from, unsigned char *to, int len) @@ -454,9 +425,6 @@ pg_johab_dsplen(const unsigned char *s) /* * convert UTF8 string to pg_wchar (UCS-4) - * caller must allocate enough space for "to", including a trailing zero! - * len: length of from. - * "from" not necessarily null terminated. */ static int pg_utf2wchar_with_len(const unsigned char *from, pg_wchar *to, int len) @@ -517,9 +485,6 @@ pg_utf2wchar_with_len(const unsigned char *from, pg_wchar *to, int len) /* * Trivial conversion from pg_wchar to UTF-8. - * caller should allocate enough space for "to" - * len: length of from. - * "from" not necessarily null terminated. */ static int pg_wchar2utf_with_len(const pg_wchar *from, unsigned char *to, int len) @@ -704,9 +669,6 @@ pg_latin12wchar_with_len(const unsigned char *from, pg_wchar *to, int len) /* * Trivial conversion from pg_wchar to single byte encoding. Just ignores * high bits. - * caller should allocate enough space for "to" - * len: length of from. - * "from" not necessarily null terminated. */ static int pg_wchar2single_with_len(const pg_wchar *from, unsigned char *to, int len) @@ -891,25 +853,9 @@ pg_gb18030_dsplen(const unsigned char *s) /* *------------------------------------------------------------------- * multibyte sequence validators - * - * The verifychar functions accept "s", a pointer to the first byte of a - * string, and "len", the remaining length of the string. If there is a - * validly encoded character beginning at *s, return its length in bytes; - * else return -1. - * - * The verifystr functions also accept "s", a pointer to a string and "len", - * the length of the string. They verify the whole string, and return the - * number of input bytes (<= len) that are valid. In other words, if the - * whole string is valid, verifystr returns "len", otherwise it returns the - * byte offset of the first invalid character. The verifystr functions must - * test for and reject zeroes in the input. - * - * The verifychar functions can assume that len > 0 and that *s != '\0', but - * they must test for and reject zeroes in any additional bytes of a - * multibyte character. Note that this definition allows the function for a - * single-byte encoding to be just "return 1". *------------------------------------------------------------------- */ + static int pg_ascii_verifychar(const unsigned char *s, int len) { @@ -1982,7 +1928,7 @@ pg_encoding_dsplen(int encoding, const char *mbstr) /* * Verify the first multibyte character of the given string. - * Return its byte length if good, -1 if bad. (See comments above for + * Return its byte length if good, -1 if bad. (See comments at pg_wchar_tbl for * full details of the mbverifychar API.) */ int @@ -1996,7 +1942,7 @@ pg_encoding_verifymbchar(int encoding, const char *mbstr, int len) /* * Verify that a string is valid for the given encoding. * Returns the number of input bytes (<= len) that form a valid string. - * (See comments above for full details of the mbverifystr API.) + * (See comments at pg_wchar_tbl for full details of the mbverifystr API.) */ int pg_encoding_verifymbstr(int encoding, const char *mbstr, int len) diff --git a/src/include/mb/pg_wchar.h b/src/include/mb/pg_wchar.h index 9f0e4223748..72d960bde6d 100644 --- a/src/include/mb/pg_wchar.h +++ b/src/include/mb/pg_wchar.h @@ -208,35 +208,115 @@ extern PGDLLIMPORT const char *pg_enc2gettext_tbl[]; /* * pg_wchar stuff */ -typedef int (*mb2wchar_with_len_converter) (const unsigned char *from, - pg_wchar *to, - int len); -typedef int (*wchar2mb_with_len_converter) (const pg_wchar *from, - unsigned char *to, - int len); +typedef int (*mblen_function) (const unsigned char *mbstr); +typedef int (*mbverifychar_function) (const unsigned char *mbstr, int len); -typedef int (*mblen_converter) (const unsigned char *mbstr); +/* + * Operations on multi-byte encodings are driven by a table of helper + * functions. + * + * All functions are required, except the conversion functions + * .mb2wchar_with_len and .wchar2mb_with_len, which are only required for + * server encodings. + * + * These functions generally assume that their input is validly formed, and + * unless specified otherwise have no special return value for errors. The + * exceptions are the two "verify" functions, which must check their input and + * signal invalid input through their return value. None of these functions + * report errors via ereport(). + */ +typedef struct +{ + /* + * Convert a multibyte string to a pg_wchar string. + * + * "len" is the length of "from" in bytes (not characters). The input + * string need not be zero-terminated, but conversion will stop early on a + * zero byte. The output is written to "to". "to" must be allocated to + * accommodate at least len + 1 pg_wchar's. A terminating 0 is always + * written. The return value is the number of pg_wchar's written, without + * the terminator. + * + * If a multibyte sequence at the end is cut off by len, it is silently + * dropped. + */ + int (*mb2wchar_with_len) (const unsigned char *from, pg_wchar *to, int len); -typedef int (*mbdisplaylen_converter) (const unsigned char *mbstr); + /* + * Convert a pg_wchar string to a multibyte string. + * + * "len" is the length of "from" in pg_wchar characters (not bytes). The + * input need not be terminated by a 0 pg_wchar, but conversion will stop + * early on a 0 pg_wchar. The output is written to "to". "to" must be + * allocated as len * maxmblen + 1 bytes. A terminating 0 byte is always + * written. The return value is the number of bytes written, without the + * terminator. + */ + int (*wchar2mb_with_len) (const pg_wchar *from, unsigned char *to, int len); -typedef bool (*mbcharacter_incrementer) (unsigned char *mbstr, int len); + /* + * Get byte length of the first character of the input string. + * + * If the input string has length zero, then 1 is returned. The return + * value is always between 1 and maxmblen inclusive. + * + * We expect that mblen does not need to examine more than the first byte + * of the character to discover the correct length. GB18030 is an + * exception; see pg_gb18030_mblen(). + */ + mblen_function mblen; -typedef int (*mbchar_verifier) (const unsigned char *mbstr, int len); + /* + * Get the display width of the first character of the input string. + * + * The possible results are -1, 0, 1 and 2. Control characters return -1; + * callers can handle that specially. Unlike mblen, an empty string + * returns 0. (But the converse is not true: Zero-width characters also + * return 0.) + * + * Note: for the display output of psql to work properly, the return + * values of the dsplen functions must conform to the Unicode standard. In + * particular the NUL character is zero width and control characters are + * generally width -1. It is recommended that non-ASCII encodings refer + * their ASCII subset to the ASCII routines to ensure consistency. + */ + int (*dsplen) (const unsigned char *mbstr); -typedef int (*mbstr_verifier) (const unsigned char *mbstr, int len); + /* + * Verify multibyte character. + * + * The verifychar functions accept "mbstr", a pointer to the first byte of + * a string, and "len", the remaining length of the string. If there is a + * validly encoded character beginning at *mbstr, return its length in + * bytes; else return -1. + * + * The verifychar functions can assume that len > 0 and that *mbstr != + * '\0', but they must test for and reject zeroes in any additional bytes + * of a multibyte character. Note that this definition allows the + * function for a single-byte encoding to be just "return 1". + */ + mbverifychar_function mbverifychar; -typedef struct -{ - mb2wchar_with_len_converter mb2wchar_with_len; /* convert a multibyte - * string to a wchar */ - wchar2mb_with_len_converter wchar2mb_with_len; /* convert a wchar string - * to a multibyte */ - mblen_converter mblen; /* get byte length of a char */ - mbdisplaylen_converter dsplen; /* get display width of a char */ - mbchar_verifier mbverifychar; /* verify multibyte character */ - mbstr_verifier mbverifystr; /* verify multibyte string */ - int maxmblen; /* max bytes for a char in this encoding */ + /* + * Verify multibyte string. + * + * The verifystr functions accept "mbstr", a pointer to a string and + * "len", the length of the string. They verify the whole string, and + * return the number of input bytes (<= len) that are valid. In other + * words, if the whole string is valid, verifystr returns "len", otherwise + * it returns the byte offset of the first invalid character. The + * verifystr functions must test for and reject zeroes in the input. + */ + int (*mbverifystr) (const unsigned char *mbstr, int len); + + /* + * The maximum number of bytes a character in this encoding could occupy. + * + * This is used to size output buffers of conversion routines, so it must + * not be too small. + */ + int maxmblen; } pg_wchar_tbl; extern PGDLLIMPORT const pg_wchar_tbl pg_wchar_table[]; @@ -547,6 +627,7 @@ extern int pg_encoding_mbcliplen(int encoding, const char *mbstr, int len, int limit); extern int pg_mbcharcliplen(const char *mbstr, int len, int limit); extern int pg_database_encoding_max_length(void); +typedef bool (*mbcharacter_incrementer) (unsigned char *mbstr, int len); extern mbcharacter_incrementer pg_database_encoding_character_incrementer(void); extern int PrepareClientEncoding(int encoding); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index cfef572df80..7393e27a534 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -3952,12 +3952,9 @@ manifest_wal_range manifest_writer map_variable_attnos_context max_parallel_hazard_context -mb2wchar_with_len_converter -mbchar_verifier mbcharacter_incrementer -mbdisplaylen_converter -mblen_converter -mbstr_verifier +mblen_function +mbverifychar_function memoize_hash memoize_iterator metastring @@ -4440,7 +4437,6 @@ walrcv_receive_fn walrcv_send_fn walrcv_server_version_fn walrcv_startstreaming_fn -wchar2mb_with_len_converter wchar_t win32_deadchild_waitinfo wint_t -- 2.56.0