From d8405b1f34c826353933108f82d5ffc8a8082b53 Mon Sep 17 00:00:00 2001 From: Thomas Munro Date: Sat, 30 Nov 2024 20:00:19 +1300 Subject: [PATCH v1] Formalize the encoding of text in shared catalogs. The encoding of shared catalogs was previously undefined. Each database assumed its own encoding, and early connection phases worked with raw bytes and hoped for the best. Database names, role names, and more could be corrupted in various scenarios. This commit introduces a new cluster-wide property to track and enforce the shared catalog encoding. It is selected at initdb time, and can be changed later with the ALTER SYSTEM CATALOG ENCODING command. The chosen encoding imposes one of three different behaviors: 1. SQL_ASCII: Databases are allowed to use different encodings, but database names, role names etc are restricted to 7-bit ASCII. 2. Any other encoding: All databases and the shared catalogs use the same encoding. Database names, role names etc can use non-ASCII characters, but databases cannot be created with different encodings. 3. UNKNOWN: Traditional behavior, supported for backwards compatibility. Not recommended, and may be removed in a future release. Work in progress! Discussion: https://postgr.es/m/CA%2BhUKGKKNAc599Vp7kFAnLE1%3DV%3DceYujz_YQoSNrvNFGaJ6i7w%40mail.gmail.com --- doc/src/sgml/catalogs.sgml | 2 + doc/src/sgml/charset.sgml | 66 ++- doc/src/sgml/ref/alter_system.sgml | 43 +- doc/src/sgml/ref/initdb.sgml | 25 + src/backend/access/rmgrdesc/xlogdesc.c | 10 + src/backend/access/transam/xlog.c | 42 +- src/backend/bootstrap/bootstrap.c | 9 +- src/backend/catalog/Makefile | 1 + src/backend/catalog/encoding.c | 429 ++++++++++++++++++ src/backend/catalog/meson.build | 1 + src/backend/catalog/pg_db_role_setting.c | 5 + src/backend/commands/alter.c | 18 + src/backend/commands/comment.c | 3 + src/backend/commands/dbcommands.c | 28 ++ src/backend/commands/seclabel.c | 3 + src/backend/commands/subscriptioncmds.c | 6 + src/backend/commands/tablespace.c | 4 + src/backend/commands/user.c | 4 + src/backend/parser/gram.y | 7 + src/backend/replication/logical/origin.c | 2 + src/backend/tcop/utility.c | 11 +- src/bin/initdb/initdb.c | 28 +- src/bin/pg_controldata/pg_controldata.c | 3 + src/bin/pg_upgrade/pg_upgrade.c | 19 + src/bin/psql/tab-complete.in.c | 11 +- src/bin/scripts/t/020_createdb.pl | 4 +- src/include/access/xlog.h | 5 +- src/include/access/xlog_internal.h | 6 + src/include/catalog/catalog.h | 2 + src/include/catalog/pg_control.h | 5 +- src/include/commands/alter.h | 2 + src/include/nodes/parsenodes.h | 1 + .../regress/expected/catalog_encoding.out | 171 +++++++ .../regress/expected/catalog_encoding_1.out | 3 + src/test/regress/parallel_schedule | 2 + src/test/regress/sql/catalog_encoding.sql | 138 ++++++ src/tools/pgindent/typedefs.list | 1 + 37 files changed, 1107 insertions(+), 13 deletions(-) create mode 100644 src/backend/catalog/encoding.c create mode 100644 src/test/regress/expected/catalog_encoding.out create mode 100644 src/test/regress/expected/catalog_encoding_1.out create mode 100644 src/test/regress/sql/catalog_encoding.sql diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index 59bb833f48d..49deeae0b5f 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -36,6 +36,8 @@ database creation and are thereafter database-specific. A few catalogs are physically shared across all databases in a cluster; these are noted in the descriptions of the individual catalogs. + This has implications for their character set; see + for details. diff --git a/doc/src/sgml/charset.sgml b/doc/src/sgml/charset.sgml index 00e1986849a..8b082ed4c56 100644 --- a/doc/src/sgml/charset.sgml +++ b/doc/src/sgml/charset.sgml @@ -2222,7 +2222,9 @@ initdb -E EUC_JP You can specify a non-default encoding at database creation time, - provided that the encoding is compatible with the selected locale: + provided that the encoding is compatible with the selected locale + and the shared catalog encoding is SQL_ASCII + (or UNKNOWN, not recommended). createdb -E EUC_KR -T template0 --lc-collate=ko_KR.euckr --lc-ctype=ko_KR.euckr korean @@ -2401,6 +2403,68 @@ RESET client_encoding; + + The Character Set used in Shared Catalogs + + The names of databases, roles and a small number of other object type + are + stored in shared system catalogs, + and are visible from all the databases in a cluster. + Since databases can use different encodings, a trade-off is required to + maintain compatibility with the encoding used in the shared catalogs. + There are two main configurations: + + + + + Single-encoding clusters: To allow non-ASCII characters to be used in + database and role names, all databases must use the same encoding and + the shared catalog encoding must agree. By default, + initdb creates clusters this way. + + + + + Multi-encoding clusters: For databases with different + encodings to co-exist, the shared catalog encoding must be + SQL_ASCII. Database and role names are restricted + to the ASCII character set, in this context meaning strict 7-bit ASCII. + ASCII is a subset of all supported server encodings, so + conforming strings are also valid in every database's encoding without + conversion (or risk of conversion failure). + + + + + + A third configuration is supported, but not recommened for new deployements: + The shared catalog encoding can be set to the special value + UNKNOWN + to allow databases with different encodings to co-exist while also allowing + non-ASCII characters in database and role names. In this configuration, + corruption may occur due to encoding mismatch. (This is the traditional + behavior of PostgreSQL releases before version 18.) + + + The shared catalog encoding can be selected with the + option to initdb, or + changed later using the + command. Later changes are only possible if certain conditions are met. For + SQL_ASCII, shared catalogs must contain no non-ASCII + characters, or an error message will identify the object preventing the change. + For any other encoding, all databases must be using that encoding. + Changing to UNKNOWN is always possible without restriction, + but not recommended. + + + Note that when shared catalog encoding is set to SQL_ASCII, + it only affects the names and properties of a small number of kinds of database + objects that have cluster-wide visibility. Most objects such as tables, + indexes and functions are stored in per-database catalogs and can always + use the full character set of the database's encoding. + + + Available Character Set Conversions diff --git a/doc/src/sgml/ref/alter_system.sgml b/doc/src/sgml/ref/alter_system.sgml index 1bde66d6ad2..e8fa5ebe890 100644 --- a/doc/src/sgml/ref/alter_system.sgml +++ b/doc/src/sgml/ref/alter_system.sgml @@ -25,6 +25,7 @@ ALTER SYSTEM SET configuration_parameterconfiguration_parameter ALTER SYSTEM RESET ALL +ALTER SYSTEM CATALOG ENCODING { encoding | UNKNOWN } @@ -32,7 +33,7 @@ ALTER SYSTEM RESET ALL Description - ALTER SYSTEM is used for changing server configuration + ALTER SYSTEM { SET | RESET } is used for changing server configuration parameters across the entire database cluster. It can be more convenient than the traditional method of manually editing the postgresql.conf file. @@ -46,7 +47,7 @@ ALTER SYSTEM RESET ALL - Values set with ALTER SYSTEM will be effective after + Values set with ALTER SYSTEM { SET | RESET } will be effective after the next server configuration reload, or after the next server restart in the case of parameters that can only be changed at server start. A server configuration reload can be commanded by calling the SQL @@ -54,6 +55,15 @@ ALTER SYSTEM RESET ALL or sending a SIGHUP signal to the main server process. + + ALTER SYSTEM CATALOG ENCODING is used to control the encoding of + the shared catalogs, and affects the character set used for role names, + database names and the properties of certain other database objects that are shared + by all databases. See for details. + Unlike ALTER SYSTEM SET, this command takes effect immediately, if + the required conditions required are met. + + Only superusers and users granted ALTER SYSTEM privilege on a parameter can change it using ALTER SYSTEM. Also, since @@ -96,6 +106,19 @@ ALTER SYSTEM RESET ALL + + + encoding + + + SQL_ASCII for the 7-bit ASCII character set. + The only other acceptable encoding is the one used by the current + database (since it must match all databases, it must at least match + this one). + + + + @@ -137,6 +160,22 @@ ALTER SYSTEM SET wal_level = replica; ALTER SYSTEM RESET wal_level; + + + Change the shared catalog encoding to UTF8, so that + non-ASCII role names and database names can be used: + +ALTER SYSTEM CATALOG ENCODING UTF8; +CREATE USER françois; + + + Try to change the shared catalog encoding to SQL_ASCII: + +ALTER SYSTEM CATALOG ENCODING SQL_ASCII; +ERROR: existing role name "françois" contains invalid characters +HINT: Consider ALTER ROLE ... RENAME TO ... using characters valid in SQL_ASCII. + + diff --git a/doc/src/sgml/ref/initdb.sgml b/doc/src/sgml/ref/initdb.sgml index 0c32114cf70..6185cc47393 100644 --- a/doc/src/sgml/ref/initdb.sgml +++ b/doc/src/sgml/ref/initdb.sgml @@ -132,6 +132,10 @@ PostgreSQL documentation To alter the default encoding, use the . + To specify a different encoding for the shared catalogs with + ; this affects the character + set available for database and role names, and the ability to create + databases with different encodings. More details can be found in . @@ -227,6 +231,27 @@ PostgreSQL documentation + + + + + + Selects the encoding of the shared catalogs. + + + By default, shared catalog encoding is set to match the default database + encoding, meaning that database and role names can use that full character + set but all databases must use that encoding. Alternatively, + SQL_ASCII can be specified to restrict the + character set for those object names to 7-bit ASCII, but allow databases + with different encodings. A third option UNKNOWN + is alway available, but not recommend. The shared catalog + encoding can be changed later, under certain conditions. + See for more details. + + + + diff --git a/src/backend/access/rmgrdesc/xlogdesc.c b/src/backend/access/rmgrdesc/xlogdesc.c index 363294d6234..3bbf0f2c9f1 100644 --- a/src/backend/access/rmgrdesc/xlogdesc.c +++ b/src/backend/access/rmgrdesc/xlogdesc.c @@ -134,6 +134,13 @@ xlog_desc(StringInfo buf, XLogReaderState *record) xlrec.wal_log_hints ? "on" : "off", xlrec.track_commit_timestamp ? "on" : "off"); } + else if (info == XLOG_CATALOG_ENCODING_CHANGE) + { + xl_catalog_encoding_change xlrec; + + memcpy(&xlrec, rec, sizeof(xl_catalog_encoding_change)); + appendStringInfo(buf, "encoding=%d", xlrec.encoding); + } else if (info == XLOG_FPW_CHANGE) { bool fpw; @@ -197,6 +204,9 @@ xlog_identify(uint8 info) case XLOG_PARAMETER_CHANGE: id = "PARAMETER_CHANGE"; break; + case XLOG_CATALOG_ENCODING_CHANGE: + id = "CATALOG_ENCODING_CHANGE"; + break; case XLOG_RESTORE_POINT: id = "RESTORE_POINT"; break; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 6f58412bcab..5f2fa0c0472 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -70,6 +70,7 @@ #include "common/file_utils.h" #include "executor/instrument.h" #include "miscadmin.h" +#include "mb/pg_wchar.h" #include "pg_trace.h" #include "pgstat.h" #include "port/atomics.h" @@ -5030,7 +5031,8 @@ XLOGShmemInit(void) * and the initial XLOG segment. */ void -BootStrapXLOG(uint32 data_checksum_version) +BootStrapXLOG(int shared_catalog_encoding, + uint32 data_checksum_version) { CheckPoint checkPoint; char *buffer; @@ -5173,6 +5175,7 @@ BootStrapXLOG(uint32 data_checksum_version) /* Now create pg_control */ InitControlFile(sysidentifier, data_checksum_version); + ControlFile->shared_catalog_encoding = shared_catalog_encoding; ControlFile->time = checkPoint.time; ControlFile->checkPoint = checkPoint.redo; ControlFile->checkPointCopy = checkPoint; @@ -8557,6 +8560,13 @@ xlog_redo(XLogReaderState *record) /* Check to see if any parameter change gives a problem on recovery */ CheckRequiredParameterValues(); } + else if (info == XLOG_CATALOG_ENCODING_CHANGE) + { + xl_catalog_encoding_change xlrec; + + memcpy(&xlrec, XLogRecGetData(record), sizeof(xlrec)); + ControlFile->shared_catalog_encoding = xlrec.encoding; + } else if (info == XLOG_FPW_CHANGE) { bool fpw; @@ -9510,3 +9520,33 @@ SetWalWriterSleeping(bool sleeping) XLogCtl->WalWriterSleeping = sleeping; SpinLockRelease(&XLogCtl->info_lck); } + +/* + * Get the shared catalog encoding. This should only be called when a lock is + * held on one of the shared catalog tables. + */ +int +GetSharedCatalogEncoding(void) +{ + return ControlFile->shared_catalog_encoding; +} + +/* + * Set the shared catalog encoding. This should only be called with + * AccessExclusiveLock held on *all* shared catalog tables that contain names. + */ +void +SetSharedCatalogEncoding(int encoding) +{ + xl_catalog_encoding_change xlrec; + + START_CRIT_SECTION(); + MyProc->delayChkptFlags |= DELAY_CHKPT_START; + xlrec.encoding = encoding; + XLogBeginInsert(); + XLogRegisterData((char *) &xlrec, sizeof(xlrec)); + XLogFlush(XLogInsert(RM_XLOG_ID, XLOG_CATALOG_ENCODING_CHANGE)); + ControlFile->shared_catalog_encoding = encoding; + MyProc->delayChkptFlags &= ~DELAY_CHKPT_START; + END_CRIT_SECTION(); +} diff --git a/src/backend/bootstrap/bootstrap.c b/src/backend/bootstrap/bootstrap.c index d31a67599c9..f6ee9752afa 100644 --- a/src/backend/bootstrap/bootstrap.c +++ b/src/backend/bootstrap/bootstrap.c @@ -202,6 +202,7 @@ BootstrapModeMain(int argc, char *argv[], bool check_only) int flag; char *userDoption = NULL; uint32 bootstrap_data_checksum_version = 0; /* No checksum */ + int shared_catalog_encoding = -1; Assert(!IsUnderPostmaster); @@ -217,7 +218,7 @@ BootstrapModeMain(int argc, char *argv[], bool check_only) argv++; argc--; - while ((flag = getopt(argc, argv, "B:c:d:D:Fkr:X:-:")) != -1) + while ((flag = getopt(argc, argv, "B:c:C:d:D:Fkr:X:-:")) != -1) { switch (flag) { @@ -266,6 +267,9 @@ BootstrapModeMain(int argc, char *argv[], bool check_only) pfree(debugstr); } break; + case 'C': + shared_catalog_encoding = atoi(optarg); + break; case 'F': SetConfigOption("fsync", "false", PGC_POSTMASTER, PGC_S_ARGV); break; @@ -341,7 +345,8 @@ BootstrapModeMain(int argc, char *argv[], bool check_only) BaseInit(); bootstrap_signals(); - BootStrapXLOG(bootstrap_data_checksum_version); + BootStrapXLOG(shared_catalog_encoding, + bootstrap_data_checksum_version); /* * To ensure that src/common/link-canary.c is linked into the backend, we diff --git a/src/backend/catalog/Makefile b/src/backend/catalog/Makefile index 1589a75fd53..ed460a95e62 100644 --- a/src/backend/catalog/Makefile +++ b/src/backend/catalog/Makefile @@ -17,6 +17,7 @@ OBJS = \ aclchk.o \ catalog.o \ dependency.o \ + encoding.o \ heap.o \ index.o \ indexing.o \ diff --git a/src/backend/catalog/encoding.c b/src/backend/catalog/encoding.c new file mode 100644 index 00000000000..f8a1b95a536 --- /dev/null +++ b/src/backend/catalog/encoding.c @@ -0,0 +1,429 @@ +/*------------------------------------------------------------------------- + * + * encoding.c + * Shared catalog encoding management. + * + * + * Portions Copyright (c) 2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * + * IDENTIFICATION + * src/backend/catalog/encoding.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "access/genam.h" +#include "access/table.h" +#include "access/xlog.h" +#include "catalog/catalog.h" +#include "catalog/pg_authid.h" +#include "catalog/pg_database.h" +#include "catalog/pg_db_role_setting.h" +#include "catalog/pg_namespace.h" +#include "catalog/pg_parameter_acl.h" +#include "catalog/pg_replication_origin.h" +#include "catalog/pg_shdescription.h" +#include "catalog/pg_shseclabel.h" +#include "catalog/pg_subscription.h" +#include "catalog/pg_tablespace.h" +#include "commands/dbcommands.h" +#include "common/string.h" +#include "mb/pg_wchar.h" +#include "miscadmin.h" +#include "storage/lmgr.h" +#include "utils/builtins.h" + +/* + * Check if a NULL-terminated string can be inserted into a shared catalog. + * The caller must hold a lock on the shared catalog table, to block + * AlterSystemCatalogEncoding(). + */ +void +ValidateSharedCatalogString(Relation rel, const char *s) +{ + /* + * The main reason for taking the rel argument is to make sure that caller + * remembered to lock the catalog before validating strings to be + * inserted. But we might as well check it's a shared relation since we + * have it. + */ + Assert(rel->rd_rel->relisshared); + + /* + * If using SQL_ASCII, then we have to make sure this string is clean + * 7-bit ASCII, so that it is valid in every supported encoding. + */ + if (GetSharedCatalogEncoding() != PG_SQL_ASCII) + { + /* + * Otherwise, either we're in UNKNOWN mode where anything goes, or all + * databases are using the same encoding and matches the shared + * catalog encoding. We don't have to validate anything. + */ + Assert(GetSharedCatalogEncoding() == -1 || + GetSharedCatalogEncoding() == GetDatabaseEncoding()); + return; + } + + if (!pg_is_ascii(s)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), + errmsg("the string \"%s\" contains characters that cannot be represented in SQL_ASCII", + s), + errhint("Consider ALTER SYSTEM CATALOG ENCODING %s.", + pg_encoding_to_char(GetDatabaseEncoding())))); +} + +/* + * Try to change the shared catalog encoding, if all the conditions are met. + */ +void +AlterSystemCatalogEncoding(const char *encoding_name) +{ + Relation rel; + SysScanDesc scan; + HeapTuple tup; + int encoding; + + if (!superuser()) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("permission denied"))); + + /* Decode the name. */ + if (pg_strcasecmp(encoding_name, "UNKNOWN") == 0) + encoding = -1; + else + { + /* + * The only encodings that can really work are SQL_ASCII and the + * current database's encoding, but we use the standard name lookup + * pattern because it tolerates some spelling variants. + */ + encoding = pg_char_to_encoding(encoding_name); + if (encoding == -1) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_OBJECT), + errmsg("%s is not a valid encoding name", + encoding_name))); + if (!PG_VALID_BE_ENCODING(encoding)) + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("invalid server encoding %d", + encoding))); + } + + /* + * Lock all of the shared catalog tables containing name or text values, + * to prevent concurrent updates. This is the set of shared catalogs that + * contain text. If new shared catalogs are invented that hold text they + * will need to be handled here too. For every validation that we perform + * below, there must also be corresponding calls to + * ValidateSharedCatalogString() in the commands that CREATE or ALTER + * these database objects. + */ + LockRelationOid(AuthIdRelationId, AccessExclusiveLock); + LockRelationOid(DatabaseRelationId, AccessExclusiveLock); + LockRelationOid(DbRoleSettingRelationId, AccessExclusiveLock); + LockRelationOid(ParameterAclRelationId, AccessExclusiveLock); + LockRelationOid(ReplicationOriginRelationId, AccessExclusiveLock); + LockRelationOid(SharedDescriptionRelationId, AccessExclusiveLock); + LockRelationOid(SharedSecLabelRelationId, AccessExclusiveLock); + LockRelationOid(SubscriptionRelationId, AccessExclusiveLock); + LockRelationOid(TableSpaceRelationId, AccessExclusiveLock); + + /* No change? */ + if (GetSharedCatalogEncoding() == encoding) + return; + + if (encoding == -1) + { + /* There are no encoding restrictions for UNKNOWN. Good luck. */ + } + else if (encoding == PG_SQL_ASCII) + { + /* Make sure all shared catalogs contain only pure 7-bit ASCII. */ + + /* pg_authid */ + rel = table_open(AuthIdRelationId, NoLock); + scan = systable_beginscan(rel, InvalidOid, false, NULL, 0, NULL); + while ((tup = systable_getnext(scan))) + { + Form_pg_authid authid = (Form_pg_authid) GETSTRUCT(tup); + + if (!pg_is_ascii(NameStr(authid->rolname))) + ereport(ERROR, + (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), + errmsg("existing role name \"%s\" contains invalid characters", + NameStr(authid->rolname)), + errhint("Consider ALTER ROLE ... RENAME TO ... using characters valid in SQL_ASCII."))); + } + systable_endscan(scan); + table_close(rel, NoLock); + + /* pg_database */ + rel = table_open(DatabaseRelationId, NoLock); + scan = systable_beginscan(rel, InvalidOid, false, NULL, 0, NULL); + while ((tup = systable_getnext(scan))) + { + Form_pg_database db = (Form_pg_database) GETSTRUCT(tup); + + if (!pg_is_ascii(NameStr(db->datname))) + ereport(ERROR, + (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), + errmsg("existing database name \"%s\" contains invalid characters", + NameStr(db->datname)), + errhint("Consider ALTER DATABASE ... RENAME TO ... using characters valid in SQL_ASCII."))); + + /* + * Locale-related text fields requiring heap tuple deforming + * should already have been validated as pure ASCII, so we don't + * have to work harder here. + * + * XXX That's only true for the LC_ stuff; what about ICU, should + * it get the same treatment, or be checked here? + */ + } + systable_endscan(scan); + table_close(rel, NoLock); + + /* pg_db_role_setting */ + rel = table_open(DbRoleSettingRelationId, NoLock); + scan = systable_beginscan(rel, InvalidOid, false, NULL, 0, NULL); + while ((tup = systable_getnext(scan))) + { + bool isnull; + Datum setconfig; + + setconfig = heap_getattr(tup, Anum_pg_db_role_setting_setconfig, + RelationGetDescr(rel), &isnull); + if (!isnull) + { + List *gucNames; + List *gucValues; + ListCell *lc1; + ListCell *lc2; + + TransformGUCArray(DatumGetArrayTypeP(setconfig), &gucNames, &gucValues); + forboth(lc1, gucNames, lc2, gucValues) + { + char *name = lfirst(lc1); + char *value = lfirst(lc2); + + if (!pg_is_ascii(name) || !pg_is_ascii(value)) + { + Datum db_id; + Datum role_id; + + db_id = heap_getattr(tup, Anum_pg_db_role_setting_setdatabase, + RelationGetDescr(rel), &isnull); + role_id = heap_getattr(tup, Anum_pg_db_role_setting_setrole, + RelationGetDescr(rel), &isnull); + + if (DatumGetObjectId(db_id) == InvalidOid) + ereport(ERROR, + (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), + errmsg("role \"%s\" has setting \"%s\" with value \"%s\" that contains invalid characters", + GetUserNameFromId(DatumGetObjectId(role_id), false), + name, + value), + errhint("Consider ALTER ROLE ... SET ... TO ... using characters valid in SQL_ASCII."))); + else + ereport(ERROR, + (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), + errmsg("role \"%s\" has setting \"%s\" with value \"%s\" in database \"%s\" that contains invalid characters", + GetUserNameFromId(DatumGetObjectId(role_id), false), + name, + value, + get_database_name(DatumGetObjectId(db_id))), + errhint("Consider ALTER ROLE ... IN DATABASE ... SET ... TO ... using characters valid in SQL_ASCII."))); + } + pfree(name); + pfree(value); + } + list_free(gucNames); + list_free(gucValues); + } + } + systable_endscan(scan); + table_close(rel, NoLock); + + /* pg_parameter_acl */ + rel = table_open(ParameterAclRelationId, NoLock); + scan = systable_beginscan(rel, InvalidOid, false, NULL, 0, NULL); + while ((tup = systable_getnext(scan))) + { + bool isnull; + char *parname; + + parname = TextDatumGetCString(heap_getattr(tup, Anum_pg_parameter_acl_parname, + RelationGetDescr(rel), &isnull)); + + /* + * This probably shouldn't happen as they are GUC names, so it's + * hard to suggest a useful hint. + */ + if (!pg_is_ascii(parname)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), + errmsg("existing ACL parameter name name \"%s\" contains invalid characters", + parname))); + pfree(parname); + } + systable_endscan(scan); + table_close(rel, NoLock); + + /* pg_replication_origin */ + rel = table_open(ReplicationOriginRelationId, NoLock); + scan = systable_beginscan(rel, InvalidOid, false, NULL, 0, NULL); + while ((tup = systable_getnext(scan))) + { + bool isnull; + char *s; + + s = TextDatumGetCString(heap_getattr(tup, Anum_pg_replication_origin_roname, + RelationGetDescr(rel), &isnull)); + if (!pg_is_ascii(s)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), + errmsg("replication origin \"%s\" contains invalid characters", + s), + errhint("Consider recreating the replication origin using characters valid in SQL_ASCII."))); + pfree(s); + } + systable_endscan(scan); + table_close(rel, NoLock); + + /* pg_shdescription */ + rel = table_open(SharedDescriptionRelationId, NoLock); + scan = systable_beginscan(rel, InvalidOid, false, NULL, 0, NULL); + while ((tup = systable_getnext(scan))) + { + bool isnull; + char *s; + + s = TextDatumGetCString(heap_getattr(tup, Anum_pg_shdescription_description, + RelationGetDescr(rel), &isnull)); + if (!pg_is_ascii(s)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), + errmsg("comment \"%s\" on a shared database object contains invalid characters", + s), + errhint("Consider COMMENT ON ... IS ... using characters valid in SQL_ASCII."))); + pfree(s); + } + systable_endscan(scan); + table_close(rel, NoLock); + + /* pg_shseclabel */ + rel = table_open(SharedSecLabelRelationId, NoLock); + scan = systable_beginscan(rel, InvalidOid, false, NULL, 0, NULL); + while ((tup = systable_getnext(scan))) + { + bool isnull; + char *s; + + s = TextDatumGetCString(heap_getattr(tup, Anum_pg_shseclabel_provider, + RelationGetDescr(rel), &isnull)); + if (!pg_is_ascii(s)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), + errmsg("security label provider name \"%s\" contains invalid characters", + s), + errhint("This security label provider cannot be used for shared database object with SQL_ASCII encoding."))); + pfree(s); + + s = TextDatumGetCString(heap_getattr(tup, Anum_pg_shseclabel_label, + RelationGetDescr(rel), &isnull)); + if (!pg_is_ascii(s)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), + errmsg("a security label on a shared database object contains invalid characters"), + errhint("Security labels applied to shared database objects must be representable in the new encoding."))); + pfree(s); + } + systable_endscan(scan); + table_close(rel, NoLock); + + /* pg_subscription */ + rel = table_open(SubscriptionRelationId, NoLock); + scan = systable_beginscan(rel, InvalidOid, false, NULL, 0, NULL); + while ((tup = systable_getnext(scan))) + { + bool isnull; + char *name; + char *s; + + name = NameStr(*DatumGetName(heap_getattr(tup, Anum_pg_subscription_subname, + RelationGetDescr(rel), &isnull))); + if (!pg_is_ascii(name)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), + errmsg("existing subscription name \"%s\" contains invalid characters", + name), + errhint("Consider ALTER SUBSCRIPTION ... RENAME TO ... using characters valid in SQL_ASCII."))); + + s = TextDatumGetCString(heap_getattr(tup, Anum_pg_subscription_subconninfo, + RelationGetDescr(rel), &isnull)); + if (!pg_is_ascii(s)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), + errmsg("existing subscription \"%s\" has connection string \"%s\" containing invalid characters", + name, s), + errhint("Consider ALTER SUBSCRIPTION ... CONNECTION ... using characters valid in SQL_ASCII."))); + pfree(s); + + /* + * subsynccommit, subslotname and suborigin have their own + * validation that requires ASCII, so no check for now. + */ + + /* XXX TODO check subpublications, a text[] */ + } + systable_endscan(scan); + table_close(rel, NoLock); + + /* pg_tablespace */ + rel = table_open(TableSpaceRelationId, NoLock); + scan = systable_beginscan(rel, InvalidOid, false, NULL, 0, NULL); + while ((tup = systable_getnext(scan))) + { + Form_pg_tablespace ts = (Form_pg_tablespace) GETSTRUCT(tup); + + if (!pg_is_ascii(NameStr(ts->spcname))) + ereport(ERROR, + (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), + errmsg("existing tablespace name \"%s\" contains invalid characters", + NameStr(ts->spcname)), + errhint("Consider ALTER TABLESPACE ... RENAME TO ... using characters valid in SQL_ASCII."))); + } + systable_endscan(scan); + table_close(rel, NoLock); + } + else + { + /* Make sure all databases are using this encoding. */ + rel = table_open(DatabaseRelationId, NoLock); + scan = systable_beginscan(rel, InvalidOid, false, NULL, 0, NULL); + while ((tup = systable_getnext(scan))) + { + Form_pg_database db = (Form_pg_database) GETSTRUCT(tup); + + if (db->encoding != encoding) + ereport(ERROR, + (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), + errmsg("database \"%s\" has incompatible encoding %s", + NameStr(db->datname), + pg_encoding_to_char(db->encoding)))); + } + systable_endscan(scan); + table_close(rel, NoLock); + } + + /* If we made it this far, we are allowed to change it. */ + SetSharedCatalogEncoding(encoding); +} diff --git a/src/backend/catalog/meson.build b/src/backend/catalog/meson.build index 2f3ded8a0e7..ba7ac5ce35f 100644 --- a/src/backend/catalog/meson.build +++ b/src/backend/catalog/meson.build @@ -4,6 +4,7 @@ backend_sources += files( 'aclchk.c', 'catalog.c', 'dependency.c', + 'encoding.c', 'heap.c', 'index.c', 'indexing.c', diff --git a/src/backend/catalog/pg_db_role_setting.c b/src/backend/catalog/pg_db_role_setting.c index 8c20f519fc0..ef81eb95d17 100644 --- a/src/backend/catalog/pg_db_role_setting.c +++ b/src/backend/catalog/pg_db_role_setting.c @@ -46,6 +46,11 @@ AlterSetting(Oid databaseid, Oid roleid, VariableSetStmt *setstmt) NULL, 2, scankey); tuple = systable_getnext(scan); + if (setstmt->name) + ValidateSharedCatalogString(rel, setstmt->name); + if (valuestr) + ValidateSharedCatalogString(rel, valuestr); + /* * There are three cases: * diff --git a/src/backend/commands/alter.c b/src/backend/commands/alter.c index a45f3bb6b83..8591f1e9bcf 100644 --- a/src/backend/commands/alter.c +++ b/src/backend/commands/alter.c @@ -1038,3 +1038,21 @@ AlterObjectOwner_internal(Oid classId, Oid objectId, Oid new_ownerId) table_close(rel, RowExclusiveLock); } + +/* + * Main entry point for ALTER SYSTEM command. + */ +void +AlterSystem(AlterSystemStmt *stmt) +{ + if (stmt->setstmt) + { + /* ALTER SYSTEM [RE]SET ... */ + AlterSystemSetConfigFile(stmt); + } + else if (stmt->encoding_name) + { + /* ALTER SYSTEM CATALOG ENCODING ... */ + AlterSystemCatalogEncoding(stmt->encoding_name); + } +} diff --git a/src/backend/commands/comment.c b/src/backend/commands/comment.c index e9d50fc7d87..f56d8624827 100644 --- a/src/backend/commands/comment.c +++ b/src/backend/commands/comment.c @@ -277,6 +277,9 @@ CreateSharedComments(Oid oid, Oid classoid, const char *comment) shdescription = table_open(SharedDescriptionRelationId, RowExclusiveLock); + if (comment) + ValidateSharedCatalogString(shdescription, comment); + sd = systable_beginscan(shdescription, SharedDescriptionObjIndexId, true, NULL, 2, skey); diff --git a/src/backend/commands/dbcommands.c b/src/backend/commands/dbcommands.c index aa91a396967..3a1cec8be3d 100644 --- a/src/backend/commands/dbcommands.c +++ b/src/backend/commands/dbcommands.c @@ -1429,6 +1429,32 @@ createdb(ParseState *pstate, const CreatedbStmt *stmt) Assert((dblocprovider != COLLPROVIDER_LIBC && dblocale) || (dblocprovider == COLLPROVIDER_LIBC && !dblocale)); + /* + * Check encoding of strings going into shared catalog. Locales have + * already been verified as ASCII by checklocale() so we skip those. + */ + ValidateSharedCatalogString(pg_database_rel, dbname); + if (dblocale) + ValidateSharedCatalogString(pg_database_rel, dblocale); + if (dbicurules) + ValidateSharedCatalogString(pg_database_rel, dbicurules); + if (dbcollversion) + ValidateSharedCatalogString(pg_database_rel, dbcollversion); + + /* + * Check encoding of the contents of the data, for compatibility with the + * shared catalogs. + */ + if (GetSharedCatalogEncoding() != -1 && + GetSharedCatalogEncoding() != PG_SQL_ASCII && + GetSharedCatalogEncoding() != encoding) + ereport(ERROR, + (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), + errmsg("encoding \"%s\" is not compatible with shared catalog encoding \"%s\"", + pg_encoding_to_char(encoding), + pg_encoding_to_char(GetSharedCatalogEncoding())), + errhint("Consider ALTER SYSTEM CATALOG ENCODING SQL_ASCII."))); + /* Form tuple */ new_record[Anum_pg_database_oid - 1] = ObjectIdGetDatum(dboid); new_record[Anum_pg_database_datname - 1] = @@ -1889,6 +1915,8 @@ RenameDatabase(const char *oldname, const char *newname) */ rel = table_open(DatabaseRelationId, RowExclusiveLock); + ValidateSharedCatalogString(rel, newname); + if (!get_db_info(oldname, AccessExclusiveLock, &db_id, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL)) ereport(ERROR, diff --git a/src/backend/commands/seclabel.c b/src/backend/commands/seclabel.c index 5607273bf9f..0dad1a09eac 100644 --- a/src/backend/commands/seclabel.c +++ b/src/backend/commands/seclabel.c @@ -363,6 +363,9 @@ SetSharedSecurityLabel(const ObjectAddress *object, pg_shseclabel = table_open(SharedSecLabelRelationId, RowExclusiveLock); + ValidateSharedCatalogString(pg_shseclabel, provider); + ValidateSharedCatalogString(pg_shseclabel, label); + scan = systable_beginscan(pg_shseclabel, SharedSecLabelObjectIndexId, true, NULL, 3, keys); diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c index 03e97730e73..5fac449328c 100644 --- a/src/backend/commands/subscriptioncmds.c +++ b/src/backend/commands/subscriptioncmds.c @@ -552,6 +552,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, bits32 supported_opts; SubOpts opts = {0}; AclResult aclresult; + ListCell *l; /* * Parse and check options. @@ -619,6 +620,11 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, rel = table_open(SubscriptionRelationId, RowExclusiveLock); + ValidateSharedCatalogString(rel, stmt->subname); + ValidateSharedCatalogString(rel, stmt->conninfo); + foreach(l, stmt->publication) + ValidateSharedCatalogString(rel, strVal(lfirst(l))); + /* Check if name is used */ subid = GetSysCacheOid2(SUBSCRIPTIONNAME, Anum_pg_subscription_oid, MyDatabaseId, CStringGetDatum(stmt->subname)); diff --git a/src/backend/commands/tablespace.c b/src/backend/commands/tablespace.c index 8ebbd935b0c..f7105ce5bd3 100644 --- a/src/backend/commands/tablespace.c +++ b/src/backend/commands/tablespace.c @@ -311,6 +311,8 @@ CreateTableSpace(CreateTableSpaceStmt *stmt) */ rel = table_open(TableSpaceRelationId, RowExclusiveLock); + ValidateSharedCatalogString(rel, stmt->tablespacename); + if (IsBinaryUpgrade) { /* Use binary-upgrade override for tablespace oid */ @@ -941,6 +943,8 @@ RenameTableSpace(const char *oldname, const char *newname) /* Search pg_tablespace */ rel = table_open(TableSpaceRelationId, RowExclusiveLock); + ValidateSharedCatalogString(rel, newname); + ScanKeyInit(&entry[0], Anum_pg_tablespace_spcname, BTEqualStrategyNumber, F_NAMEEQ, diff --git a/src/backend/commands/user.c b/src/backend/commands/user.c index e7ade898a47..02fae91c8d0 100644 --- a/src/backend/commands/user.c +++ b/src/backend/commands/user.c @@ -371,6 +371,8 @@ CreateRole(ParseState *pstate, CreateRoleStmt *stmt) pg_authid_rel = table_open(AuthIdRelationId, RowExclusiveLock); pg_authid_dsc = RelationGetDescr(pg_authid_rel); + ValidateSharedCatalogString(pg_authid_rel, stmt->role); + if (OidIsValid(get_role_oid(stmt->role, true))) ereport(ERROR, (errcode(ERRCODE_DUPLICATE_OBJECT), @@ -1350,6 +1352,8 @@ RenameRole(const char *oldname, const char *newname) rel = table_open(AuthIdRelationId, RowExclusiveLock); dsc = RelationGetDescr(rel); + ValidateSharedCatalogString(rel, newname); + oldtuple = SearchSysCache1(AUTHNAME, CStringGetDatum(oldname)); if (!HeapTupleIsValid(oldtuple)) ereport(ERROR, diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index 67eb96396af..53a0a826497 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -11540,6 +11540,13 @@ AlterSystemStmt: n->setstmt = $4; $$ = (Node *) n; } + | ALTER SYSTEM_P CATALOG_P ENCODING name + { + AlterSystemStmt *n = makeNode(AlterSystemStmt); + + n->encoding_name = $5; + $$ = (Node *) n; + } ; diff --git a/src/backend/replication/logical/origin.c b/src/backend/replication/logical/origin.c index baf696d8e68..0a323ea2422 100644 --- a/src/backend/replication/logical/origin.c +++ b/src/backend/replication/logical/origin.c @@ -286,6 +286,8 @@ replorigin_create(const char *roname) rel = table_open(ReplicationOriginRelationId, ExclusiveLock); + ValidateSharedCatalogString(rel, roname); + for (roident = InvalidOid + 1; roident < PG_UINT16_MAX; roident++) { bool nulls[Natts_pg_replication_origin]; diff --git a/src/backend/tcop/utility.c b/src/backend/tcop/utility.c index f28bf371059..e6e6e6ab824 100644 --- a/src/backend/tcop/utility.c +++ b/src/backend/tcop/utility.c @@ -218,6 +218,8 @@ ClassifyUtilityCommandAsReadOnly(Node *parsetree) case T_AlterSystemStmt: { + AlterSystemStmt *stmt = (AlterSystemStmt *) parsetree; + /* * Surprisingly, ALTER SYSTEM meets all our definitions of * read-only: it changes nothing that affects the output of @@ -227,8 +229,13 @@ ClassifyUtilityCommandAsReadOnly(Node *parsetree) * * So, despite the fact that it writes to a file, it's read * only! + * + * XXX ^ that's about ALTER SYSTEM SET only */ - return COMMAND_IS_STRICTLY_READ_ONLY; + if (stmt->setstmt) + return COMMAND_IS_STRICTLY_READ_ONLY; + else + return COMMAND_IS_NOT_READ_ONLY; } case T_CallStmt: @@ -868,7 +875,7 @@ standard_ProcessUtility(PlannedStmt *pstmt, case T_AlterSystemStmt: PreventInTransactionBlock(isTopLevel, "ALTER SYSTEM"); - AlterSystemSetConfigFile((AlterSystemStmt *) parsetree); + AlterSystem((AlterSystemStmt *) parsetree); break; case T_VariableSetStmt: diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c index 9a91830783e..423ecdf7d2b 100644 --- a/src/bin/initdb/initdb.c +++ b/src/bin/initdb/initdb.c @@ -137,6 +137,7 @@ static char *share_path = NULL; /* values to be obtained from arguments */ static char *pg_data = NULL; static char *encoding = NULL; +static char *catalog_encoding = NULL; static char *locale = NULL; static char *lc_collate = NULL; static char *lc_ctype = NULL; @@ -173,6 +174,7 @@ static DataDirSyncMethod sync_method = DATA_DIR_SYNC_METHOD_FSYNC; /* internal vars */ static const char *progname; static int encodingid; +static int catalog_encodingid; static char *bki_file; static char *hba_file; static char *ident_file; @@ -1593,6 +1595,7 @@ bootstrap_template1(void) printfPQExpBuffer(&cmd, "\"%s\" --boot %s %s", backend_exec, boot_options, extra_options); appendPQExpBuffer(&cmd, " -X %d", wal_segment_size_mb * (1024 * 1024)); + appendPQExpBuffer(&cmd, " -C %d", catalog_encodingid); if (data_checksums) appendPQExpBuffer(&cmd, " -k"); if (debug) @@ -2748,6 +2751,25 @@ setup_locale_encoding(void) else encodingid = get_encoding_id(encoding); + /* Choose initial value of shared_catalog_encoding. */ + if (catalog_encoding == NULL) + catalog_encodingid = encodingid; + else if (pg_strcasecmp(catalog_encoding, "UNKNOWN") == 0) + catalog_encodingid = -1; + else if (pg_strcasecmp(catalog_encoding, "SQL_ASCII") == 0) + catalog_encodingid = PG_SQL_ASCII; + else if (pg_char_to_encoding(catalog_encoding) != encodingid) + { + pg_log_error("catalog encoding \"%s\" is invalid", catalog_encoding); + pg_log_error_detail("Valid values are UNKNOWN, SQL_ASCII or %s (the selected default database encoding)", + pg_encoding_to_char(encodingid)); + exit(1); + } + else + catalog_encodingid = encodingid; + printf(_("The initial shared catalog encoding has been set to \"%s\".\n"), + catalog_encodingid == -1 ? "UNKNOWN " : pg_encoding_to_char(catalog_encodingid)); + if (!check_locale_encoding(lc_ctype, encodingid) || !check_locale_encoding(lc_collate, encodingid)) exit(1); /* check_locale_encoding printed the error */ @@ -3146,6 +3168,7 @@ main(int argc, char *argv[]) static struct option long_options[] = { {"pgdata", required_argument, NULL, 'D'}, {"encoding", required_argument, NULL, 'E'}, + {"catalog-encoding", required_argument, NULL, 'C'}, {"locale", required_argument, NULL, 1}, {"lc-collate", required_argument, NULL, 2}, {"lc-ctype", required_argument, NULL, 3}, @@ -3224,7 +3247,7 @@ main(int argc, char *argv[]) /* process command-line options */ - while ((c = getopt_long(argc, argv, "A:c:dD:E:gkL:nNsST:U:WX:", + while ((c = getopt_long(argc, argv, "A:c:C:dD:E:gkL:nNsST:U:WX:", long_options, &option_index)) != -1) { switch (c) @@ -3272,6 +3295,9 @@ main(int argc, char *argv[]) case 'E': encoding = pg_strdup(optarg); break; + case 'C': + catalog_encoding = pg_strdup(optarg); + break; case 'W': pwprompt = true; break; diff --git a/src/bin/pg_controldata/pg_controldata.c b/src/bin/pg_controldata/pg_controldata.c index 93a05d80ca7..4284cdda303 100644 --- a/src/bin/pg_controldata/pg_controldata.c +++ b/src/bin/pg_controldata/pg_controldata.c @@ -27,6 +27,7 @@ #include "common/controldata_utils.h" #include "common/logging.h" #include "getopt_long.h" +#include "mb/pg_wchar.h" #include "pg_getopt.h" static void @@ -325,6 +326,8 @@ main(int argc, char *argv[]) (ControlFile->float8ByVal ? _("by value") : _("by reference"))); printf(_("Data page checksum version: %u\n"), ControlFile->data_checksum_version); + printf(_("Shared catalog encoding: %d\n"), + ControlFile->shared_catalog_encoding); printf(_("Mock authentication nonce: %s\n"), mock_auth_nonce_str); return 0; diff --git a/src/bin/pg_upgrade/pg_upgrade.c b/src/bin/pg_upgrade/pg_upgrade.c index 663235816f8..72d9b1d5a39 100644 --- a/src/bin/pg_upgrade/pg_upgrade.c +++ b/src/bin/pg_upgrade/pg_upgrade.c @@ -467,6 +467,25 @@ set_locale_and_encoding(void) PQfreemem(datctype_literal); PQfreemem(datlocale_literal); + /* + * Set the shared catalog encoding to UNKNOWN, because we have to tolerate + * multi-encoding source clusters that didn't adhere to our new rules about + * non-ASCII in database/role/etc names. + * + * TODO: for source >= 18, here we should just copy the source's cat + * encoding instead. + * + * TODO: for source < 18, we should start here with UNKNOWN, and then at + * end try to change it to template0's encoding or SQL_ASCII in some order + * (which?), to see if the validation passes. We'd gradually and + * automatically upgrade the world's clusters to defined shared catalog + * encoding. Or at least the easy cases that already conform to our new + * rules, anyway. The rest would have to do manual work to escape from + * UNKNOWN. + */ + PQclear(executeQueryOrDie(conn_new_template1, + "ALTER SYSTEM CATALOG ENCODING UNKNOWN")); + PQfinish(conn_new_template1); check_ok(); diff --git a/src/bin/psql/tab-complete.in.c b/src/bin/psql/tab-complete.in.c index bbd08770c3d..2d1fc4a483b 100644 --- a/src/bin/psql/tab-complete.in.c +++ b/src/bin/psql/tab-complete.in.c @@ -1009,6 +1009,13 @@ static const SchemaQuery Query_for_trigger_of_table = { " FROM pg_catalog.pg_conversion "\ " WHERE pg_catalog.pg_encoding_to_char(conforencoding) LIKE pg_catalog.upper('%s')" +#define Query_for_list_of_catalog_encodings \ +"SELECT 'SQL_ASCII' "\ +" UNION ALL "\ +"SELECT pg_catalog.getdatabaseencoding() "\ +" UNION ALL "\ +"SELECT 'UNKNOWN'" + #define Query_for_list_of_languages \ "SELECT lanname "\ " FROM pg_catalog.pg_language "\ @@ -2550,7 +2557,9 @@ match_previous_words(int pattern_id, COMPLETE_WITH("OPTIONS"); /* ALTER SYSTEM SET, RESET, RESET ALL */ else if (Matches("ALTER", "SYSTEM")) - COMPLETE_WITH("SET", "RESET"); + COMPLETE_WITH("SET", "RESET", "CATALOG ENCODING"); + else if (Matches("ALTER", "SYSTEM", "CATALOG", "ENCODING")) + COMPLETE_WITH_QUERY_VERBATIM(Query_for_list_of_catalog_encodings); else if (Matches("ALTER", "SYSTEM", "SET|RESET")) COMPLETE_WITH_QUERY_VERBATIM_PLUS(Query_for_list_of_alter_system_set_vars, "ALL"); diff --git a/src/bin/scripts/t/020_createdb.pl b/src/bin/scripts/t/020_createdb.pl index 4a0e2c883a1..f68b7d56cf7 100644 --- a/src/bin/scripts/t/020_createdb.pl +++ b/src/bin/scripts/t/020_createdb.pl @@ -12,8 +12,10 @@ program_help_ok('createdb'); program_version_ok('createdb'); program_options_handling_ok('createdb'); +# Because we're using different encodings in the same cluster, we need shared +# catalog encoding set to SQL_ASCII for this test. my $node = PostgreSQL::Test::Cluster->new('main'); -$node->init; +$node->init(extra => [ '--catalog-encoding=SQL_ASCII' ]); $node->start; $node->issues_sql_like( diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h index 34ad46c067b..39e0325b35a 100644 --- a/src/include/access/xlog.h +++ b/src/include/access/xlog.h @@ -234,7 +234,8 @@ extern bool DataChecksumsEnabled(void); extern XLogRecPtr GetFakeLSNForUnloggedRel(void); extern Size XLOGShmemSize(void); extern void XLOGShmemInit(void); -extern void BootStrapXLOG(uint32 data_checksum_version); +extern void BootStrapXLOG(int shared_catalog_encoding, + uint32 data_checksum_version); extern void InitializeWalConsistencyChecking(void); extern void LocalProcessControlFile(bool reset); extern WalLevel GetActiveWalLevelOnStandby(void); @@ -253,6 +254,8 @@ extern XLogRecPtr GetFlushRecPtr(TimeLineID *insertTLI); extern TimeLineID GetWALInsertionTimeLine(void); extern TimeLineID GetWALInsertionTimeLineIfSet(void); extern XLogRecPtr GetLastImportantRecPtr(void); +extern int GetSharedCatalogEncoding(void); +extern void SetSharedCatalogEncoding(int encoding); extern void SetWalWriterSleeping(bool sleeping); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d9cf51a0f9f..bf4075e3886 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -305,6 +305,12 @@ typedef struct xl_end_of_recovery int wal_level; } xl_end_of_recovery; +/* Change to the encoding of shared catalogs. */ +typedef struct xl_catalog_encoding_change +{ + int encoding; +} xl_catalog_encoding_change; + /* * The functions in xloginsert.c construct a chain of XLogRecData structs * to represent the final WAL record. diff --git a/src/include/catalog/catalog.h b/src/include/catalog/catalog.h index a8dd304b1ad..36a0e1625b1 100644 --- a/src/include/catalog/catalog.h +++ b/src/include/catalog/catalog.h @@ -43,5 +43,7 @@ extern Oid GetNewOidWithIndex(Relation relation, Oid indexId, extern RelFileNumber GetNewRelFileNumber(Oid reltablespace, Relation pg_class, char relpersistence); +extern void ValidateSharedCatalogString(Relation rel, const char *s); +extern void AlterSystemCatalogEncoding(const char *encoding_name); #endif /* CATALOG_H */ diff --git a/src/include/catalog/pg_control.h b/src/include/catalog/pg_control.h index e80ff8e4140..b21af726c5b 100644 --- a/src/include/catalog/pg_control.h +++ b/src/include/catalog/pg_control.h @@ -77,7 +77,7 @@ typedef struct CheckPoint #define XLOG_END_OF_RECOVERY 0x90 #define XLOG_FPI_FOR_HINT 0xA0 #define XLOG_FPI 0xB0 -/* 0xC0 is used in Postgres 9.5-11 */ +#define XLOG_CATALOG_ENCODING_CHANGE 0xC0 #define XLOG_OVERWRITE_CONTRECORD 0xD0 #define XLOG_CHECKPOINT_REDO 0xE0 @@ -221,6 +221,9 @@ typedef struct ControlFileData /* Are data pages protected by checksums? Zero if no checksum version */ uint32 data_checksum_version; + /* A pg_enc value, or -1 for UNKNOWN. */ + int shared_catalog_encoding; + /* * Random nonce, used in authentication requests that need to proceed * based on values that are cluster-unique, like a SASL exchange that diff --git a/src/include/commands/alter.h b/src/include/commands/alter.h index f00af75beff..295d6726ebc 100644 --- a/src/include/commands/alter.h +++ b/src/include/commands/alter.h @@ -31,4 +31,6 @@ extern ObjectAddress ExecAlterOwnerStmt(AlterOwnerStmt *stmt); extern void AlterObjectOwner_internal(Oid classId, Oid objectId, Oid new_ownerId); +extern void AlterSystem(AlterSystemStmt *stmt); + #endif /* ALTER_H */ diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index 0f9462493e3..5698783571f 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -3841,6 +3841,7 @@ typedef struct AlterSystemStmt { NodeTag type; VariableSetStmt *setstmt; /* SET subcommand */ + const char *encoding_name; /* CATALOG ENCODING subcommand */ } AlterSystemStmt; /* ---------------------- diff --git a/src/test/regress/expected/catalog_encoding.out b/src/test/regress/expected/catalog_encoding.out new file mode 100644 index 00000000000..4ccc1e69325 --- /dev/null +++ b/src/test/regress/expected/catalog_encoding.out @@ -0,0 +1,171 @@ +SELECT getdatabaseencoding() <> 'UTF8' AS skip_test \gset +\if :skip_test +\quit +\endif +-- Check that we are blocked from putting non-ASCII into the shared +-- catalogs by all known routes, when the encoding is SQL_ASCII. This +-- exercises the ValidateSharedCatalogString() calls that should cover +-- all entry points. +ALTER SYSTEM CATALOG ENCODING SQL_ASCII; +-- pg_authid +CREATE USER regress_astérix; +ERROR: the string "regress_astérix" contains characters that cannot be represented in SQL_ASCII +HINT: Consider ALTER SYSTEM CATALOG ENCODING UTF8. +CREATE USER regress_fred; +ALTER USER regress_fred RENAME TO regress_astérix; +ERROR: the string "regress_astérix" contains characters that cannot be represented in SQL_ASCII +HINT: Consider ALTER SYSTEM CATALOG ENCODING UTF8. +DROP USER regress_fred; +-- pg_database +CREATE DATABASE regression_café; +ERROR: the string "regression_café" contains characters that cannot be represented in SQL_ASCII +HINT: Consider ALTER SYSTEM CATALOG ENCODING UTF8. +ALTER DATABASE template1 RENAME TO regression_café; +ERROR: the string "regression_café" contains characters that cannot be represented in SQL_ASCII +HINT: Consider ALTER SYSTEM CATALOG ENCODING UTF8. +CREATE DATABASE regression_ok TEMPLATE template0 LOCALE 'français'; +WARNING: locale name "français" contains non-ASCII characters +ERROR: invalid LC_COLLATE locale name: "français" +HINT: If the locale name is specific to ICU, use ICU_LOCALE. +-- pg_db_role_setting +CREATE USER regress_fred; +ALTER ROLE regress_fred SET application_name TO 'café'; +ERROR: the string "café" contains characters that cannot be represented in SQL_ASCII +HINT: Consider ALTER SYSTEM CATALOG ENCODING UTF8. +DROP USER regress_fred; +-- pg_parameter_acl +-- XXX +-- pg_replication_origin +SELECT pg_replication_origin_create('regress_café'); +ERROR: the string "regress_café" contains characters that cannot be represented in SQL_ASCII +HINT: Consider ALTER SYSTEM CATALOG ENCODING UTF8. +-- pg_shdescription +COMMENT ON DATABASE template0 IS 'café'; +ERROR: the string "café" contains characters that cannot be represented in SQL_ASCII +HINT: Consider ALTER SYSTEM CATALOG ENCODING UTF8. +-- non-shared objects are OK, because non-shared catalog +COMMENT ON TABLE pg_catalog.pg_class IS 'café'; +COMMENT ON TABLE pg_catalog.pg_class IS NULL; +-- pg_shseclabel +-- XXX +-- pg_subscription +CREATE SUBSCRIPTION regress_café CONNECTION 'dbname=crême' PUBLICATION brûlée; +ERROR: the string "regress_café" contains characters that cannot be represented in SQL_ASCII +HINT: Consider ALTER SYSTEM CATALOG ENCODING UTF8. +CREATE SUBSCRIPTION regress_ok CONNECTION 'dbname=crême' PUBLICATION brûlée; +ERROR: the string "dbname=crême" contains characters that cannot be represented in SQL_ASCII +HINT: Consider ALTER SYSTEM CATALOG ENCODING UTF8. +CREATE SUBSCRIPTION regress_ok CONNECTION 'dbname=ok' PUBLICATION brûlée; +ERROR: the string "brûlée" contains characters that cannot be represented in SQL_ASCII +HINT: Consider ALTER SYSTEM CATALOG ENCODING UTF8. +CREATE SUBSCRIPTION regress_ok CONNECTION 'dbname=ok' PUBLICATION ok WITH (slot_name = 'café'); +ERROR: replication slot name "café" contains invalid character +HINT: Replication slot names may only contain lower case letters, numbers, and the underscore character. +CREATE SUBSCRIPTION regress_ok CONNECTION 'dbname=ok' PUBLICATION ok WITH (synchronous_commit = 'café'); +ERROR: invalid value for parameter "synchronous_commit": "café" +HINT: Available values: local, remote_write, remote_apply, on, off. +CREATE SUBSCRIPTION regress_ok CONNECTION 'dbname=ok' PUBLICATION ok WITH (origin = 'café'); +ERROR: unrecognized origin value: "café" +-- pg_tablespace +SET allow_in_place_tablespaces = 'on'; +CREATE TABLESPACE regress_café LOCATION ''; +ERROR: the string "regress_café" contains characters that cannot be represented in SQL_ASCII +HINT: Consider ALTER SYSTEM CATALOG ENCODING UTF8. +CREATE TABLESPACE regress_ok LOCATION ''; +ALTER TABLESPACE regress_ok RENAME TO regress_café; +ERROR: the string "regress_café" contains characters that cannot be represented in SQL_ASCII +HINT: Consider ALTER SYSTEM CATALOG ENCODING UTF8. +DROP TABLESPACE regress_ok; +-- Check that we can create a new database with a different encoding, +-- while the shared catalog encoding is SQL_ASCII +CREATE DATABASE regression_latin1 TEMPLATE template0 LOCALE 'C' ENCODING 'LATIN1'; +-- Check that we can't change the shared catalog encoding to UTF8, becaus that +-- LATIN1 database is in the way, then drop it so we can. +ALTER SYSTEM CATALOG ENCODING UTF8; +ERROR: database "regression_latin1" has incompatible encoding LATIN1 +DROP DATABASE regression_latin1; +ALTER SYSTEM CATALOG ENCODING UTF8; +-- Test that we can now do each of those things that failed before, and that +-- those things block us from going back to SQL_ASCII. +-- pg_authid +CREATE USER regress_astérix; +ALTER SYSTEM CATALOG ENCODING SQL_ASCII; +ERROR: existing role name "regress_astérix" contains invalid characters +HINT: Consider ALTER ROLE ... RENAME TO ... using characters valid in SQL_ASCII. +DROP USER regress_astérix; +-- pg_database +CREATE DATABASE regression_café; +ALTER SYSTEM CATALOG ENCODING SQL_ASCII; +ERROR: existing database name "regression_café" contains invalid characters +HINT: Consider ALTER DATABASE ... RENAME TO ... using characters valid in SQL_ASCII. +DROP DATABASE regression_café; +-- but we can't make a LATIN1 database while we have UTF8 catalogs +CREATE DATABASE regression_latin1 TEMPLATE template0 LOCALE 'C' ENCODING 'LATIN1'; +ERROR: encoding "LATIN1" is not compatible with shared catalog encoding "UTF8" +HINT: Consider ALTER SYSTEM CATALOG ENCODING SQL_ASCII. +-- pg_db_role_setting +CREATE USER regress_fred; +ALTER ROLE regress_fred SET application_name TO 'café'; +ALTER SYSTEM CATALOG ENCODING SQL_ASCII; +ERROR: role "regress_fred" has setting "application_name" with value "café" that contains invalid characters +HINT: Consider ALTER ROLE ... SET ... TO ... using characters valid in SQL_ASCII. +DROP USER regress_fred; +-- pg_parameter_acl +-- XXX +-- pg_replication_origin +SELECT pg_replication_origin_create('regress_café'); + pg_replication_origin_create +------------------------------ + 1 +(1 row) + +ALTER SYSTEM CATALOG ENCODING SQL_ASCII; +ERROR: replication origin "regress_café" contains invalid characters +HINT: Consider recreating the replication origin using characters valid in SQL_ASCII. +SELECT pg_replication_origin_drop('regress_café'); + pg_replication_origin_drop +---------------------------- + +(1 row) + +-- pg_shdescription +COMMENT ON DATABASE template0 IS 'café'; +ALTER SYSTEM CATALOG ENCODING SQL_ASCII; +ERROR: comment "café" on a shared database object contains invalid characters +HINT: Consider COMMENT ON ... IS ... using characters valid in SQL_ASCII. +COMMENT ON DATABASE template0 IS 'unmodifiable empty database'; +-- pg_shseclabel +-- XXX +-- pg_subscription +-- XXX +-- pg_tablespace +CREATE TABLESPACE regress_café LOCATION ''; +ALTER SYSTEM CATALOG ENCODING SQL_ASCII; +ERROR: existing tablespace name "regress_café" contains invalid characters +HINT: Consider ALTER TABLESPACE ... RENAME TO ... using characters valid in SQL_ASCII. +DROP TABLESPACE regress_café; +-- We dropped everything that was in the way, so we should be able to go back +-- to SQL_ASCII now. +ALTER SYSTEM CATALOG ENCODING SQL_ASCII; +-- Try out UNKNOWN mode, which is the only way to have a non-ASCII database +-- name and mutiple encodings at the same time +ALTER SYSTEM CATALOG ENCODING UNKNOWN; +CREATE DATABASE regression_café ENCODING UTF8; +CREATE DATABASE regression_latin1 TEMPLATE template0 LOCALE 'C' ENCODING 'LATIN1'; +-- We can't switch to SQL_ASCII from this state +ALTER SYSTEM CATALOG ENCODING SQL_ASCII; +ERROR: existing database name "regression_café" contains invalid characters +HINT: Consider ALTER DATABASE ... RENAME TO ... using characters valid in SQL_ASCII. +-- We also can't switch to UTF8 from this state +ALTER SYSTEM CATALOG ENCODING UTF8; +ERROR: database "regression_latin1" has incompatible encoding LATIN1 +-- If we get rid of the LATIN1 database, we can go to UTF8 +DROP DATABASE regression_latin1; +ALTER SYSTEM CATALOG ENCODING UTF8; +-- We still can't go back to SQL_ASCII unless we also get rid of the non-ASCII +-- database name. +ALTER SYSTEM CATALOG ENCODING SQL_ASCII; +ERROR: existing database name "regression_café" contains invalid characters +HINT: Consider ALTER DATABASE ... RENAME TO ... using characters valid in SQL_ASCII. +DROP DATABASE regression_café; +ALTER SYSTEM CATALOG ENCODING SQL_ASCII; diff --git a/src/test/regress/expected/catalog_encoding_1.out b/src/test/regress/expected/catalog_encoding_1.out new file mode 100644 index 00000000000..8505c4fa552 --- /dev/null +++ b/src/test/regress/expected/catalog_encoding_1.out @@ -0,0 +1,3 @@ +SELECT getdatabaseencoding() <> 'UTF8' AS skip_test \gset +\if :skip_test +\quit diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index 81e4222d26a..63de6216408 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -83,6 +83,8 @@ test: create_table_like alter_generic alter_operator misc async dbsize merge mis # collate.linux.utf8 and collate.icu.utf8 tests cannot be run in parallel with each other test: rules psql psql_crosstab amutils stats_ext collate.linux.utf8 collate.windows.win1252 +test: catalog_encoding + # ---------- # Run these alone so they don't run out of parallel workers # select_parallel depends on create_misc diff --git a/src/test/regress/sql/catalog_encoding.sql b/src/test/regress/sql/catalog_encoding.sql new file mode 100644 index 00000000000..08918322f27 --- /dev/null +++ b/src/test/regress/sql/catalog_encoding.sql @@ -0,0 +1,138 @@ +SELECT getdatabaseencoding() <> 'UTF8' AS skip_test \gset +\if :skip_test +\quit +\endif + +-- Check that we are blocked from putting non-ASCII into the shared +-- catalogs by all known routes, when the encoding is SQL_ASCII. This +-- exercises the ValidateSharedCatalogString() calls that should cover +-- all entry points. +ALTER SYSTEM CATALOG ENCODING SQL_ASCII; + +-- pg_authid +CREATE USER regress_astérix; +CREATE USER regress_fred; +ALTER USER regress_fred RENAME TO regress_astérix; +DROP USER regress_fred; + +-- pg_database +CREATE DATABASE regression_café; +ALTER DATABASE template1 RENAME TO regression_café; +CREATE DATABASE regression_ok TEMPLATE template0 LOCALE 'français'; + +-- pg_db_role_setting +CREATE USER regress_fred; +ALTER ROLE regress_fred SET application_name TO 'café'; +DROP USER regress_fred; + +-- pg_parameter_acl +-- XXX + +-- pg_replication_origin +SELECT pg_replication_origin_create('regress_café'); + +-- pg_shdescription +COMMENT ON DATABASE template0 IS 'café'; +-- non-shared objects are OK, because non-shared catalog +COMMENT ON TABLE pg_catalog.pg_class IS 'café'; +COMMENT ON TABLE pg_catalog.pg_class IS NULL; + +-- pg_shseclabel +-- XXX + +-- pg_subscription +CREATE SUBSCRIPTION regress_café CONNECTION 'dbname=crême' PUBLICATION brûlée; +CREATE SUBSCRIPTION regress_ok CONNECTION 'dbname=crême' PUBLICATION brûlée; +CREATE SUBSCRIPTION regress_ok CONNECTION 'dbname=ok' PUBLICATION brûlée; +CREATE SUBSCRIPTION regress_ok CONNECTION 'dbname=ok' PUBLICATION ok WITH (slot_name = 'café'); +CREATE SUBSCRIPTION regress_ok CONNECTION 'dbname=ok' PUBLICATION ok WITH (synchronous_commit = 'café'); +CREATE SUBSCRIPTION regress_ok CONNECTION 'dbname=ok' PUBLICATION ok WITH (origin = 'café'); + +-- pg_tablespace +SET allow_in_place_tablespaces = 'on'; +CREATE TABLESPACE regress_café LOCATION ''; +CREATE TABLESPACE regress_ok LOCATION ''; +ALTER TABLESPACE regress_ok RENAME TO regress_café; +DROP TABLESPACE regress_ok; + +-- Check that we can create a new database with a different encoding, +-- while the shared catalog encoding is SQL_ASCII +CREATE DATABASE regression_latin1 TEMPLATE template0 LOCALE 'C' ENCODING 'LATIN1'; + +-- Check that we can't change the shared catalog encoding to UTF8, becaus that +-- LATIN1 database is in the way, then drop it so we can. +ALTER SYSTEM CATALOG ENCODING UTF8; +DROP DATABASE regression_latin1; +ALTER SYSTEM CATALOG ENCODING UTF8; + +-- Test that we can now do each of those things that failed before, and that +-- those things block us from going back to SQL_ASCII. + +-- pg_authid +CREATE USER regress_astérix; +ALTER SYSTEM CATALOG ENCODING SQL_ASCII; +DROP USER regress_astérix; + +-- pg_database +CREATE DATABASE regression_café; +ALTER SYSTEM CATALOG ENCODING SQL_ASCII; +DROP DATABASE regression_café; +-- but we can't make a LATIN1 database while we have UTF8 catalogs +CREATE DATABASE regression_latin1 TEMPLATE template0 LOCALE 'C' ENCODING 'LATIN1'; + +-- pg_db_role_setting +CREATE USER regress_fred; +ALTER ROLE regress_fred SET application_name TO 'café'; +ALTER SYSTEM CATALOG ENCODING SQL_ASCII; +DROP USER regress_fred; + +-- pg_parameter_acl +-- XXX + +-- pg_replication_origin +SELECT pg_replication_origin_create('regress_café'); +ALTER SYSTEM CATALOG ENCODING SQL_ASCII; +SELECT pg_replication_origin_drop('regress_café'); + +-- pg_shdescription +COMMENT ON DATABASE template0 IS 'café'; +ALTER SYSTEM CATALOG ENCODING SQL_ASCII; +COMMENT ON DATABASE template0 IS 'unmodifiable empty database'; + +-- pg_shseclabel +-- XXX + +-- pg_subscription +-- XXX + +-- pg_tablespace +CREATE TABLESPACE regress_café LOCATION ''; +ALTER SYSTEM CATALOG ENCODING SQL_ASCII; +DROP TABLESPACE regress_café; + +-- We dropped everything that was in the way, so we should be able to go back +-- to SQL_ASCII now. +ALTER SYSTEM CATALOG ENCODING SQL_ASCII; + +-- Try out UNKNOWN mode, which is the only way to have a non-ASCII database +-- name and mutiple encodings at the same time +ALTER SYSTEM CATALOG ENCODING UNKNOWN; +CREATE DATABASE regression_café ENCODING UTF8; +CREATE DATABASE regression_latin1 TEMPLATE template0 LOCALE 'C' ENCODING 'LATIN1'; + +-- We can't switch to SQL_ASCII from this state +ALTER SYSTEM CATALOG ENCODING SQL_ASCII; + +-- We also can't switch to UTF8 from this state +ALTER SYSTEM CATALOG ENCODING UTF8; + +-- If we get rid of the LATIN1 database, we can go to UTF8 +DROP DATABASE regression_latin1; +ALTER SYSTEM CATALOG ENCODING UTF8; + +-- We still can't go back to SQL_ASCII unless we also get rid of the non-ASCII +-- database name. +ALTER SYSTEM CATALOG ENCODING SQL_ASCII; +DROP DATABASE regression_café; +ALTER SYSTEM CATALOG ENCODING SQL_ASCII; + diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index b54428b38cd..90e9940a4dd 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -4085,6 +4085,7 @@ xl_btree_split xl_btree_unlink_page xl_btree_update xl_btree_vacuum +xl_catalog_encoding_change xl_clog_truncate xl_commit_ts_truncate xl_dbase_create_file_copy_rec -- 2.47.0