diff --git a/doc/src/sgml/syntax.sgml b/doc/src/sgml/syntax.sgml index c908e0b..e134877 100644 --- a/doc/src/sgml/syntax.sgml +++ b/doc/src/sgml/syntax.sgml @@ -189,6 +189,23 @@ UPDATE "my_table" SET "a" = 5; ampersands. The length limitation still applies. + + Quoting an identifier also makes it case-sensitive, whereas + unquoted names are always folded to lower case. For example, the + identifiers FOO, foo, and + "foo" are considered the same by + PostgreSQL, but + "Foo" and "FOO" are + different from these three and each other. (The folding of + unquoted names to lower case in PostgreSQL is + incompatible with the SQL standard, which says that unquoted names + should be folded to upper case. Thus, foo + should be equivalent to "FOO" not + "foo" according to the standard. If you want + to write portable applications you are advised to always quote a + particular name or never quote it.) + + Unicode escape in identifiers @@ -230,7 +247,8 @@ U&"d!0061t!+000061" UESCAPE '!' The escape character can be any single character other than a hexadecimal digit, the plus sign, a single quote, a double quote, or a whitespace character. Note that the escape character is - written in single quotes, not double quotes. + written in single quotes, not double quotes, + after UESCAPE. @@ -239,32 +257,18 @@ U&"d!0061t!+000061" UESCAPE '!' - The Unicode escape syntax works only when the server encoding is - UTF8. When other server encodings are used, only code - points in the ASCII range (up to \007F) can be - specified. Both the 4-digit and the 6-digit form can be used to + Either the 4-digit or the 6-digit escape form can be used to specify UTF-16 surrogate pairs to compose characters with code points larger than U+FFFF, although the availability of the 6-digit form technically makes this unnecessary. (Surrogate - pairs are not stored directly, but combined into a single - code point that is then encoded in UTF-8.) + pairs are not stored directly, but are combined into a single + code point.) - Quoting an identifier also makes it case-sensitive, whereas - unquoted names are always folded to lower case. For example, the - identifiers FOO, foo, and - "foo" are considered the same by - PostgreSQL, but - "Foo" and "FOO" are - different from these three and each other. (The folding of - unquoted names to lower case in PostgreSQL is - incompatible with the SQL standard, which says that unquoted names - should be folded to upper case. Thus, foo - should be equivalent to "FOO" not - "foo" according to the standard. If you want - to write portable applications you are advised to always quote a - particular name or never quote it.) + If the server encoding is not UTF-8, the Unicode code point identified + by one of these escape sequences is converted to the actual server + encoding; an error is reported if that's not possible. @@ -427,25 +431,11 @@ SELECT 'foo' 'bar'; It is your responsibility that the byte sequences you create, especially when using the octal or hexadecimal escapes, compose - valid characters in the server character set encoding. When the - server encoding is UTF-8, then the Unicode escapes or the + valid characters in the server character set encoding. + A useful alternative is to use Unicode escapes or the alternative Unicode escape syntax, explained - in , should be used - instead. (The alternative would be doing the UTF-8 encoding by - hand and writing out the bytes, which would be very cumbersome.) - - - - The Unicode escape syntax works fully only when the server - encoding is UTF8. When other server encodings are - used, only code points in the ASCII range (up - to \u007F) can be specified. Both the 4-digit and - the 8-digit form can be used to specify UTF-16 surrogate pairs to - compose characters with code points larger than U+FFFF, although - the availability of the 8-digit form technically makes this - unnecessary. (When surrogate pairs are used when the server - encoding is UTF8, they are first combined into a - single code point that is then encoded in UTF-8.) + in ; then the server + will check that the character conversion is possible. @@ -524,16 +514,23 @@ U&'d!0061t!+000061' UESCAPE '!' - The Unicode escape syntax works only when the server encoding is - UTF8. When other server encodings are used, only - code points in the ASCII range (up to \007F) - can be specified. Both the 4-digit and the 6-digit form can be - used to specify UTF-16 surrogate pairs to compose characters with - code points larger than U+FFFF, although the availability of the - 6-digit form technically makes this unnecessary. (When surrogate - pairs are used when the server encoding is UTF8, they - are first combined into a single code point that is then encoded - in UTF-8.) + To include the escape character in the string literally, write + it twice. + + + + Either the 4-digit or the 6-digit escape form can be used to + specify UTF-16 surrogate pairs to compose characters with code + points larger than U+FFFF, although the availability of the + 6-digit form technically makes this unnecessary. (Surrogate + pairs are not stored directly, but are combined into a single + code point.) + + + + If the server encoding is not UTF-8, the Unicode code point identified + by one of these escape sequences is converted to the actual server + encoding; an error is reported if that's not possible. @@ -546,11 +543,6 @@ U&'d!0061t!+000061' UESCAPE '!' parameter is set to off, this syntax will be rejected with an error message. - - - To include the escape character in the string literally, write it - twice. - diff --git a/src/backend/parser/parser.c b/src/backend/parser/parser.c index 1bf1144..e88a5e0 100644 --- a/src/backend/parser/parser.c +++ b/src/backend/parser/parser.c @@ -292,7 +292,7 @@ hexval(unsigned char c) return 0; /* not reached */ } -/* is Unicode code point acceptable in database's encoding? */ +/* is Unicode code point acceptable? */ static void check_unicode_value(pg_wchar c, int pos, core_yyscan_t yyscanner) { @@ -302,12 +302,6 @@ check_unicode_value(pg_wchar c, int pos, core_yyscan_t yyscanner) (errcode(ERRCODE_SYNTAX_ERROR), errmsg("invalid Unicode escape value"), scanner_errposition(pos, yyscanner))); - - if (c > 0x7F && GetDatabaseEncoding() != PG_UTF8) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("Unicode escape values cannot be used for code point values above 007F when the server encoding is not UTF8"), - scanner_errposition(pos, yyscanner))); } /* is 'escape' acceptable as Unicode escape character (UESCAPE syntax) ? */ @@ -338,18 +332,30 @@ str_udeescape(const char *str, char escape, const char *in; char *new, *out; + size_t new_len; pg_wchar pair_first = 0; /* - * This relies on the subtle assumption that a UTF-8 expansion cannot be - * longer than its escaped representation. + * Guesstimate that result will be no longer than input, but allow enough + * padding for Unicode conversion. */ - new = palloc(strlen(str) + 1); + new_len = strlen(str) + MAX_UNICODE_EQUIVALENT_STRING + 1; + new = palloc(new_len); in = str; out = new; while (*in) { + /* Enlarge string if needed */ + size_t out_dist = out - new; + + if (out_dist > new_len - (MAX_UNICODE_EQUIVALENT_STRING + 1)) + { + new_len *= 2; + new = repalloc(new, new_len); + out = new + out_dist; + } + if (in[0] == escape) { if (in[1] == escape) @@ -390,8 +396,8 @@ str_udeescape(const char *str, char escape, pair_first = unicode; else { - unicode_to_utf8(unicode, (unsigned char *) out); - out += pg_mblen(out); + pg_unicode_to_server(unicode, (unsigned char *) out); + out += strlen(out); } in += 5; } @@ -431,8 +437,8 @@ str_udeescape(const char *str, char escape, pair_first = unicode; else { - unicode_to_utf8(unicode, (unsigned char *) out); - out += pg_mblen(out); + pg_unicode_to_server(unicode, (unsigned char *) out); + out += strlen(out); } in += 8; } @@ -457,13 +463,6 @@ str_udeescape(const char *str, char escape, goto invalid_pair; *out = '\0'; - - /* - * We could skip pg_verifymbstr if we didn't process any non-7-bit-ASCII - * codes; but it's probably not worth the trouble, since this isn't likely - * to be a performance-critical path. - */ - pg_verifymbstr(new, out - new, false); return new; invalid_pair: diff --git a/src/backend/parser/scan.l b/src/backend/parser/scan.l index 84c7391..3903df8 100644 --- a/src/backend/parser/scan.l +++ b/src/backend/parser/scan.l @@ -1226,19 +1226,18 @@ process_integer_literal(const char *token, YYSTYPE *lval) static void addunicode(pg_wchar c, core_yyscan_t yyscanner) { - char buf[8]; + char buf[MAX_UNICODE_EQUIVALENT_STRING + 1]; /* See also check_unicode_value() in parser.c */ if (c == 0 || c > 0x10FFFF) yyerror("invalid Unicode escape value"); - if (c > 0x7F) - { - if (GetDatabaseEncoding() != PG_UTF8) - yyerror("Unicode escape values cannot be used for code point values above 007F when the server encoding is not UTF8"); - yyextra->saw_non_ascii = true; - } - unicode_to_utf8(c, (unsigned char *) buf); - addlit(buf, pg_mblen(buf), yyscanner); + + /* + * We expect that pg_unicode_to_server() will complain about any + * unconvertible code point, so we don't have to set saw_non_ascii. + */ + pg_unicode_to_server(c, (unsigned char *) buf); + addlit(buf, strlen(buf), yyscanner); } static unsigned char diff --git a/src/backend/utils/adt/xml.c b/src/backend/utils/adt/xml.c index 3808c30..a2d2a0b 100644 --- a/src/backend/utils/adt/xml.c +++ b/src/backend/utils/adt/xml.c @@ -2086,26 +2086,6 @@ map_sql_identifier_to_xml_name(const char *ident, bool fully_escaped, /* - * Map a Unicode codepoint into the current server encoding. - */ -static char * -unicode_to_sqlchar(pg_wchar c) -{ - char utf8string[8]; /* need room for trailing zero */ - char *result; - - memset(utf8string, 0, sizeof(utf8string)); - unicode_to_utf8(c, (unsigned char *) utf8string); - - result = pg_any_to_server(utf8string, strlen(utf8string), PG_UTF8); - /* if pg_any_to_server didn't strdup, we must */ - if (result == utf8string) - result = pstrdup(result); - return result; -} - - -/* * Map XML name to SQL identifier; see SQL/XML:2008 section 9.3. */ char * @@ -2125,10 +2105,12 @@ map_xml_name_to_sql_identifier(const char *name) && isxdigit((unsigned char) *(p + 5)) && *(p + 6) == '_') { + char cbuf[MAX_UNICODE_EQUIVALENT_STRING + 1]; unsigned int u; sscanf(p + 2, "%X", &u); - appendStringInfoString(&buf, unicode_to_sqlchar(u)); + pg_unicode_to_server(u, (unsigned char *) cbuf); + appendStringInfoString(&buf, cbuf); p += 6; } else diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c index 5d7cc74..7d90ac9 100644 --- a/src/backend/utils/mb/mbutils.c +++ b/src/backend/utils/mb/mbutils.c @@ -68,6 +68,13 @@ static FmgrInfo *ToServerConvProc = NULL; static FmgrInfo *ToClientConvProc = NULL; /* + * This variable stores the conversion function to convert from UTF-8 + * to the server encoding. It's NULL if the server encoding *is* UTF-8, + * or if we lack a conversion function for this. + */ +static FmgrInfo *Utf8ToServerConvProc = NULL; + +/* * These variables track the currently-selected encodings. */ static const pg_enc2name *ClientEncoding = &pg_enc2name_tbl[PG_SQL_ASCII]; @@ -273,6 +280,8 @@ SetClientEncoding(int encoding) void InitializeClientEncoding(void) { + int current_server_encoding; + Assert(!backend_startup_complete); backend_startup_complete = true; @@ -289,6 +298,35 @@ InitializeClientEncoding(void) pg_enc2name_tbl[pending_client_encoding].name, GetDatabaseEncodingName()))); } + + /* + * Also look up the UTF8-to-server conversion function if needed. Since + * the server encoding is fixed within any one backend process, we don't + * have to do this more than once. + */ + current_server_encoding = GetDatabaseEncoding(); + if (current_server_encoding != PG_UTF8 && + current_server_encoding != PG_SQL_ASCII) + { + Oid utf8_to_server_proc; + + Assert(IsTransactionState()); + utf8_to_server_proc = + FindDefaultConversionProc(PG_UTF8, + current_server_encoding); + /* If there's no such conversion, just leave the pointer as NULL */ + if (OidIsValid(utf8_to_server_proc)) + { + FmgrInfo *finfo; + + finfo = (FmgrInfo *) MemoryContextAlloc(TopMemoryContext, + sizeof(FmgrInfo)); + fmgr_info_cxt(utf8_to_server_proc, finfo, + TopMemoryContext); + /* Set Utf8ToServerConvProc only after data is fully valid */ + Utf8ToServerConvProc = finfo; + } + } } /* @@ -752,6 +790,73 @@ perform_default_encoding_conversion(const char *src, int len, return result; } +/* + * Convert a single Unicode code point into a string in the server encoding. + * + * The code point given by "c" is converted and stored at *s, which must + * have at least MAX_UNICODE_EQUIVALENT_STRING+1 bytes available. + * The output will have a trailing '\0'. Throws error if the conversion + * cannot be performed. + * + * Note that this relies on having previously looked up any required + * conversion function. That's partly for speed but mostly because the parser + * may call this outside any transaction, or in an aborted transaction. + */ +void +pg_unicode_to_server(pg_wchar c, unsigned char *s) +{ + unsigned char c_as_utf8[MAX_MULTIBYTE_CHAR_LEN + 1]; + int c_as_utf8_len; + int server_encoding; + + /* + * Complain if invalid Unicode code point. The choice of errcode here is + * debatable, but really our caller should have checked this anyway. + */ + if (c == 0 || c > 0x10FFFF) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("invalid Unicode code point"))); + + /* Otherwise, if it's in ASCII range, conversion is trivial */ + if (c <= 0x7F) + { + s[0] = (unsigned char) c; + s[1] = '\0'; + return; + } + + /* If the server encoding is UTF-8, we just need to reformat the code */ + server_encoding = GetDatabaseEncoding(); + if (server_encoding == PG_UTF8) + { + unicode_to_utf8(c, s); + s[pg_utf_mblen(s)] = '\0'; + return; + } + + /* For all other cases, we must have a conversion function available */ + if (Utf8ToServerConvProc == NULL) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("conversion between %s and %s is not supported", + pg_enc2name_tbl[PG_UTF8].name, + GetDatabaseEncodingName()))); + + /* Construct UTF-8 source string */ + unicode_to_utf8(c, c_as_utf8); + c_as_utf8_len = pg_utf_mblen(c_as_utf8); + c_as_utf8[c_as_utf8_len] = '\0'; + + /* Convert, or throw error if we can't */ + FunctionCall5(Utf8ToServerConvProc, + Int32GetDatum(PG_UTF8), + Int32GetDatum(server_encoding), + CStringGetDatum(c_as_utf8), + CStringGetDatum(s), + Int32GetDatum(c_as_utf8_len)); +} + /* convert a multibyte string to a wchar */ int diff --git a/src/include/mb/pg_wchar.h b/src/include/mb/pg_wchar.h index 7fb5fa4..2daf301 100644 --- a/src/include/mb/pg_wchar.h +++ b/src/include/mb/pg_wchar.h @@ -316,6 +316,15 @@ typedef enum pg_enc #define MAX_CONVERSION_GROWTH 4 /* + * Maximum byte length of the string equivalent to any one Unicode code point, + * in any backend encoding. The current value assumes that a 4-byte UTF-8 + * character might expand by MAX_CONVERSION_GROWTH, which is a huge + * overestimate. But in current usage we don't allocate large multiples of + * this, so there's little point in being stingy. + */ +#define MAX_UNICODE_EQUIVALENT_STRING 16 + +/* * Table for mapping an encoding number to official encoding name and * possibly other subsidiary data. Be careful to check encoding number * before accessing a table entry! @@ -602,6 +611,8 @@ extern char *pg_server_to_client(const char *s, int len); extern char *pg_any_to_server(const char *s, int len, int encoding); extern char *pg_server_to_any(const char *s, int len, int encoding); +extern void pg_unicode_to_server(pg_wchar c, unsigned char *s); + extern unsigned short BIG5toCNS(unsigned short big5, unsigned char *lc); extern unsigned short CNStoBIG5(unsigned short cns, unsigned char lc);