1: 231c6fb165 ! 1: 557370eabb common/jsonapi: support FRONTEND clients @@ Metadata Author: Jacob Champion ## Commit message ## - common/jsonapi: support FRONTEND clients + common/jsonapi: support libpq as a client Based on a patch by Michael Paquier. For frontend code, use PQExpBuffer instead of StringInfo. This requires us to track allocation failures so that we can return JSON_OUT_OF_MEMORY - as needed. json_errdetail() now allocates its error message inside - memory owned by the JsonLexContext, so clients don't need to worry about - freeing it. + as needed rather than exit()ing. - We can now partially revert b44669b2ca, now that json_errdetail() works - correctly. + ## src/bin/pg_combinebackup/Makefile ## +@@ src/bin/pg_combinebackup/Makefile: include $(top_builddir)/src/Makefile.global - Co-authored-by: Daniel Gustafsson + override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS) + LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils ++# TODO: fix this properly ++LDFLAGS_INTERNAL += -lpgcommon $(libpq_pgport) - ## src/bin/pg_verifybackup/t/005_bad_manifest.pl ## -@@ src/bin/pg_verifybackup/t/005_bad_manifest.pl: use Test::More; - my $tempdir = PostgreSQL::Test::Utils::tempdir; + OBJS = \ + $(WIN32RES) \ +@@ src/bin/pg_combinebackup/Makefile: OBJS = \ - test_bad_manifest('input string ended unexpectedly', -- qr/could not parse backup manifest: parsing failed/, <flags & JSONLEX_FREE_STRVAL) -- { -- pfree(lex->strval->data); -- pfree(lex->strval); -- } +- destroyStringInfo(lex->strval); + destroyStrVal(lex->strval); -+ -+ if (lex->errormsg) + + if (lex->errormsg) +- destroyStringInfo(lex->errormsg); + destroyStrVal(lex->errormsg); -+ + if (lex->flags & JSONLEX_FREE_STRUCT) pfree(lex); + else @@ src/common/jsonapi.c: json_lex_string(JsonLexContext *lex) lex->prev_token_terminator = lex->token_terminator; lex->token_terminator = s + 1; @@ src/common/jsonapi.c: report_parse_error(JsonParseContext ctx, JsonLexContext *lex) - return JSON_SUCCESS; /* silence stupider compilers */ - } - -- --#ifndef FRONTEND --/* -- * Extract the current token from a lexing context, for error reporting. -- */ --static char * --extract_token(JsonLexContext *lex) --{ -- int toklen = lex->token_terminator - lex->token_start; -- char *token = palloc(toklen + 1); -- -- memcpy(token, lex->token_start, toklen); -- token[toklen] = '\0'; -- return token; --} -- - /* - * Construct an (already translated) detail message for a JSON error. - * -- * Note that the error message generated by this routine may not be -- * palloc'd, making it unsafe for frontend code as there is no way to -- * know if this can be safely pfree'd or not. -+ * The returned allocation is either static or owned by the JsonLexContext and -+ * should not be freed. - */ char * json_errdetail(JsonParseErrorType error, JsonLexContext *lex) { -+ int toklen = lex->token_terminator - lex->token_start; -+ + if (error == JSON_OUT_OF_MEMORY) + { + /* Short circuit. Allocating anything for this case is unhelpful. */ + return _("out of memory"); + } + -+ if (lex->errormsg) + if (lex->errormsg) +- resetStringInfo(lex->errormsg); + resetStrVal(lex->errormsg); -+ else + else +- lex->errormsg = makeStringInfo(); + lex->errormsg = createStrVal(); -+ + + /* + * A helper for error messages that should print the current token. The + * format must contain exactly one %.*s specifier. + */ + #define token_error(lex, format) \ +- appendStringInfo((lex)->errormsg, _(format), \ +- (int) ((lex)->token_terminator - (lex)->token_start), \ +- (lex)->token_start); ++ appendStrVal((lex)->errormsg, _(format), \ ++ (int) ((lex)->token_terminator - (lex)->token_start), \ ++ (lex)->token_start); + switch (error) { - case JSON_SUCCESS: - /* fall through to the error code after switch */ +@@ src/common/jsonapi.c: json_errdetail(JsonParseErrorType error, JsonLexContext *lex) + token_error(lex, "Escape sequence \"\\%.*s\" is invalid."); break; - case JSON_ESCAPING_INVALID: -- return psprintf(_("Escape sequence \"\\%s\" is invalid."), -- extract_token(lex)); -+ appendStrVal(lex->errormsg, -+ _("Escape sequence \"\\%.*s\" is invalid."), -+ toklen, lex->token_start); -+ break; case JSON_ESCAPING_REQUIRED: -- return psprintf(_("Character with value 0x%02x must be escaped."), +- appendStringInfo(lex->errormsg, +- _("Character with value 0x%02x must be escaped."), - (unsigned char) *(lex->token_terminator)); + appendStrVal(lex->errormsg, + _("Character with value 0x%02x must be escaped."), + (unsigned char) *(lex->token_terminator)); -+ break; + break; case JSON_EXPECTED_END: -- return psprintf(_("Expected end of input, but found \"%s\"."), -- extract_token(lex)); -+ appendStrVal(lex->errormsg, -+ _("Expected end of input, but found \"%.*s\"."), -+ toklen, lex->token_start); -+ break; - case JSON_EXPECTED_ARRAY_FIRST: -- return psprintf(_("Expected array element or \"]\", but found \"%s\"."), -- extract_token(lex)); -+ appendStrVal(lex->errormsg, -+ _("Expected array element or \"]\", but found \"%.*s\"."), -+ toklen, lex->token_start); -+ break; - case JSON_EXPECTED_ARRAY_NEXT: -- return psprintf(_("Expected \",\" or \"]\", but found \"%s\"."), -- extract_token(lex)); -+ appendStrVal(lex->errormsg, -+ _("Expected \",\" or \"]\", but found \"%.*s\"."), -+ toklen, lex->token_start); -+ break; - case JSON_EXPECTED_COLON: -- return psprintf(_("Expected \":\", but found \"%s\"."), -- extract_token(lex)); -+ appendStrVal(lex->errormsg, -+ _("Expected \":\", but found \"%.*s\"."), -+ toklen, lex->token_start); -+ break; - case JSON_EXPECTED_JSON: -- return psprintf(_("Expected JSON value, but found \"%s\"."), -- extract_token(lex)); -+ appendStrVal(lex->errormsg, -+ _("Expected JSON value, but found \"%.*s\"."), -+ toklen, lex->token_start); -+ break; - case JSON_EXPECTED_MORE: - return _("The input string ended unexpectedly."); - case JSON_EXPECTED_OBJECT_FIRST: -- return psprintf(_("Expected string or \"}\", but found \"%s\"."), -- extract_token(lex)); -+ appendStrVal(lex->errormsg, -+ _("Expected string or \"}\", but found \"%.*s\"."), -+ toklen, lex->token_start); -+ break; - case JSON_EXPECTED_OBJECT_NEXT: -- return psprintf(_("Expected \",\" or \"}\", but found \"%s\"."), -- extract_token(lex)); -+ appendStrVal(lex->errormsg, -+ _("Expected \",\" or \"}\", but found \"%.*s\"."), -+ toklen, lex->token_start); -+ break; - case JSON_EXPECTED_STRING: -- return psprintf(_("Expected string, but found \"%s\"."), -- extract_token(lex)); -+ appendStrVal(lex->errormsg, -+ _("Expected string, but found \"%.*s\"."), -+ toklen, lex->token_start); -+ break; + token_error(lex, "Expected end of input, but found \"%.*s\"."); +@@ src/common/jsonapi.c: json_errdetail(JsonParseErrorType error, JsonLexContext *lex) case JSON_INVALID_TOKEN: -- return psprintf(_("Token \"%s\" is invalid."), -- extract_token(lex)); -+ appendStrVal(lex->errormsg, -+ _("Token \"%.*s\" is invalid."), -+ toklen, lex->token_start); -+ break; + token_error(lex, "Token \"%.*s\" is invalid."); + break; + case JSON_OUT_OF_MEMORY: + /* should have been handled above; use the error path */ + break; @@ src/common/jsonapi.c: report_parse_error(JsonParseContext ctx, JsonLexContext *l return _("\\u0000 cannot be converted to text."); case JSON_UNICODE_ESCAPE_FORMAT: @@ src/common/jsonapi.c: json_errdetail(JsonParseErrorType error, JsonLexContext *lex) - /* note: this case is only reachable in frontend not backend */ - return _("Unicode escape values cannot be used for code point values above 007F when the encoding is not UTF8."); - case JSON_UNICODE_UNTRANSLATABLE: -- /* note: this case is only reachable in backend not frontend */ -+ -+ /* -+ * note: this case is only reachable in backend not frontend. -+ * #ifdef it away so the frontend doesn't try to link against -+ * backend functionality. -+ */ -+#ifndef FRONTEND - return psprintf(_("Unicode escape value could not be translated to the server's encoding %s."), - GetDatabaseEncodingName()); -+#else -+ Assert(false); -+ break; -+#endif - case JSON_UNICODE_HIGH_SURROGATE: - return _("Unicode high surrogate must not follow a high surrogate."); - case JSON_UNICODE_LOW_SURROGATE: -@@ src/common/jsonapi.c: json_errdetail(JsonParseErrorType error, JsonLexContext *lex) - break; } + #undef token_error - /* - * We don't use a default: case, so that the compiler will warn about - * unhandled enum values. But this needs to be here anyway to cover the - * possibility of an incorrect input. - */ -- elog(ERROR, "unexpected json parse error type: %d", (int) error); -- return NULL; --} +- if (lex->errormsg->len == 0) +- appendStringInfo(lex->errormsg, +- _("unexpected json parse error type: %d"), +- (int) error); + /* Note that lex->errormsg can be NULL in FRONTEND code. */ -+ if (lex->errormsg && !lex->errormsg->data[0]) ++ if (lex->errormsg && lex->errormsg->len == 0) + { + /* + * We don't use a default: case, so that the compiler will warn about @@ src/common/jsonapi.c: json_errdetail(JsonParseErrorType error, JsonLexContext *l + * the possibility of an incorrect input. + */ + appendStrVal(lex->errormsg, -+ "unexpected json parse error type: %d", (int) error); ++ _("unexpected json parse error type: %d"), ++ (int) error); + } + +#ifdef FRONTEND + if (PQExpBufferBroken(lex->errormsg)) + return _("out of memory while constructing error description"); - #endif -+ -+ return lex->errormsg->data; -+} ++#endif + + return lex->errormsg->data; + } ## src/common/meson.build ## @@ src/common/meson.build: common_sources_frontend_static += files( @@ src/common/meson.build: foreach name, opts : pgcommon_variants 'dependencies': opts['dependencies'] + [ssl], } - ## src/common/parse_manifest.c ## -@@ src/common/parse_manifest.c: json_parse_manifest(JsonManifestParseContext *context, char *buffer, - /* Run the actual JSON parser. */ - json_error = pg_parse_json(lex, &sem); - if (json_error != JSON_SUCCESS) -- json_manifest_parse_failure(context, "parsing failed"); -+ json_manifest_parse_failure(context, json_errdetail(json_error, lex)); - if (parse.state != JM_EXPECT_EOF) - json_manifest_parse_failure(context, "manifest ended unexpectedly"); - - - ## src/common/stringinfo.c ## -@@ src/common/stringinfo.c: enlargeStringInfo(StringInfo str, int needed) - - str->maxlen = newlen; - } -+ -+void -+destroyStringInfo(StringInfo str) -+{ -+ pfree(str->data); -+ pfree(str); -+} - ## src/include/common/jsonapi.h ## @@ #ifndef JSONAPI_H @@ src/include/common/jsonapi.h: typedef struct JsonLexContext int line_number; /* line number, starting from 1 */ char *line_start; /* where that line starts within input */ - StringInfo strval; +- StringInfo errormsg; + bool parse_strval; + StrValType *strval; /* only used if parse_strval == true */ + StrValType *errormsg; } JsonLexContext; typedef JsonParseErrorType (*json_struct_action) (void *state); - - ## src/include/lib/stringinfo.h ## -@@ src/include/lib/stringinfo.h: extern void appendBinaryStringInfoNT(StringInfo str, - */ - extern void enlargeStringInfo(StringInfo str, int needed); - -+ -+extern void destroyStringInfo(StringInfo str); - #endif /* STRINGINFO_H */ 2: f78c79ea68 < -: ---------- Refactor SASL exchange to return tri-state status 3: 10b6d2a6b9 < -: ---------- Explicitly require password for SCRAM exchange 4: 2a55d9c806 ! 2: b754a2f8fb libpq: add OAUTHBEARER SASL mechanism @@ src/interfaces/libpq/Makefile: endif endif ## src/interfaces/libpq/exports.txt ## -@@ src/interfaces/libpq/exports.txt: PQsendClosePrepared 190 - PQsendClosePortal 191 - PQchangePassword 192 - PQsendPipelineSync 193 -+PQsetAuthDataHook 194 -+PQgetAuthDataHook 195 -+PQdefaultAuthDataHook 196 +@@ src/interfaces/libpq/exports.txt: PQcancelSocket 199 + PQcancelErrorMessage 200 + PQcancelReset 201 + PQcancelFinish 202 ++PQsetAuthDataHook 203 ++PQgetAuthDataHook 204 ++PQdefaultAuthDataHook 205 ## src/interfaces/libpq/fe-auth-oauth-curl.c (new) ## @@ @@ src/interfaces/libpq/fe-connect.c: static const internalPQconninfoOption PQconni {NULL, NULL, NULL, NULL, NULL, NULL, 0} @@ src/interfaces/libpq/fe-connect.c: pqDropServerData(PGconn *conn) + conn->write_failed = false; + free(conn->write_err_msg); conn->write_err_msg = NULL; - conn->be_pid = 0; - conn->be_key = 0; + /* conn->oauth_want_retry = false; TODO */ - } - + /* + * Cancel connections need to retain their be_pid and be_key across @@ src/interfaces/libpq/fe-connect.c: PQconnectPoll(PGconn *conn) case CONNECTION_NEEDED: case CONNECTION_GSS_STARTUP: @@ src/interfaces/libpq/libpq-fe.h: extern "C" /* * Option flags for PQcopyResult @@ src/interfaces/libpq/libpq-fe.h: typedef enum - CONNECTION_CONSUME, /* Consuming any extra messages. */ - CONNECTION_GSS_STARTUP, /* Negotiating GSSAPI. */ - CONNECTION_CHECK_TARGET, /* Checking target server properties. */ -- CONNECTION_CHECK_STANDBY /* Checking if server is in standby mode. */ -+ CONNECTION_CHECK_STANDBY, /* Checking if server is in standby mode. */ + CONNECTION_CHECK_TARGET, /* Internal state: checking target server + * properties. */ + CONNECTION_CHECK_STANDBY, /* Checking if server is in standby mode. */ +- CONNECTION_ALLOCATED /* Waiting for connection attempt to be ++ CONNECTION_ALLOCATED, /* Waiting for connection attempt to be + * started. */ + CONNECTION_AUTHENTICATING, /* Authentication is in progress with some + * external system. */ } ConnStatusType; @@ src/interfaces/libpq/libpq-int.h: typedef struct pg_conn_host * PGconn stores all the state data associated with a single connection * to a backend. @@ src/interfaces/libpq/libpq-int.h: struct pg_conn - char *require_auth; /* name of the expected auth method */ - char *load_balance_hosts; /* load balance over hosts */ + * cancel request, instead of being a normal + * connection that's used for queries */ + /* OAuth v2 */ + char *oauth_issuer; /* token issuer URL */ 5: 5488ac25f5 ! 3: 27251b63c1 backend: add OAUTHBEARER SASL mechanism @@ src/backend/utils/misc/guc_tables.c #include "nodes/queryjumble.h" #include "optimizer/cost.h" @@ src/backend/utils/misc/guc_tables.c: struct config_string ConfigureNamesString[] = - check_debug_io_direct, assign_debug_io_direct, NULL + check_standby_slot_names, assign_standby_slot_names, NULL }, + { 6: fdbad1976a ! 4: 78617fa9ba Introduce OAuth validator libraries @@ src/backend/utils/misc/guc_tables.c: struct config_string ConfigureNamesString[] NULL, NULL, NULL }, - ## src/bin/pg_combinebackup/Makefile ## -@@ src/bin/pg_combinebackup/Makefile: OBJS = \ - all: pg_combinebackup - - pg_combinebackup: $(OBJS) | submake-libpgport submake-libpgfeutils -- $(CC) $(CFLAGS) $^ $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X) -+ $(CC) $(CFLAGS) $^ $(LDFLAGS) $(LDFLAGS_EX) $(libpq_pgport) $(LIBS) -o $@$(X) - - install: all installdirs - $(INSTALL_PROGRAM) pg_combinebackup$(X) '$(DESTDIR)$(bindir)/pg_combinebackup$(X)' - ## src/common/Makefile ## @@ src/common/Makefile: override CPPFLAGS += -DVAL_LDFLAGS_SL="\"$(LDFLAGS_SL)\"" override CPPFLAGS += -DVAL_LIBS="\"$(LIBS)\"" 7: e1da97fb50 ! 5: 1b871542f5 Add pytest suite for OAuth @@ src/test/python/client/test_oauth.py (new) + discovery_uri = issuer + "/.well-known/openid-configuration" + access_token = secrets.token_urlsafe() + -+ sock, _ = accept( ++ sock, client = accept( + oauth_issuer=issuer, + oauth_client_id="some-id", + oauth_scope=scope, @@ src/test/python/client/test_oauth.py (new) + assert call.openid_configuration.decode() == discovery_uri + assert call.scope == (None if scope is None else scope.encode()) + -+ # Make sure we cleaned up after ourselves. ++ # Make sure we clean up after ourselves when the connection is finished. ++ client.check_completed() + assert cleanup_calls == 1 + + 8: 185f9902fd ! 6: e6c5d94682 XXX temporary patches to build and test @@ Metadata ## Commit message ## XXX temporary patches to build and test - - the new pg_combinebackup utility uses JSON in the frontend without - 0001; has something changed? - construct 2.10.70 has some incompatibilities with the current tests - temporarily skip the exit check (from Daniel Gustafsson); this needs to be turned into an exception for curl rather than a plain exit call - ## src/bin/pg_combinebackup/Makefile ## -@@ src/bin/pg_combinebackup/Makefile: include $(top_builddir)/src/Makefile.global - - override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS) - LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils -+# TODO: fix this properly -+LDFLAGS_INTERNAL += -lpgcommon $(libpq_pgport) - - OBJS = \ - $(WIN32RES) \ -@@ src/bin/pg_combinebackup/Makefile: OBJS = \ - - all: pg_combinebackup - --pg_combinebackup: $(OBJS) | submake-libpgport submake-libpgfeutils -- $(CC) $(CFLAGS) $^ $(LDFLAGS) $(LDFLAGS_EX) $(libpq_pgport) $(LIBS) -o $@$(X) -+pg_combinebackup: $(OBJS) | submake-libpq submake-libpgport submake-libpgfeutils -+ $(CC) $(CFLAGS) $^ $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X) - - install: all installdirs - $(INSTALL_PROGRAM) pg_combinebackup$(X) '$(DESTDIR)$(bindir)/pg_combinebackup$(X)' - - ## src/bin/pg_combinebackup/meson.build ## -@@ src/bin/pg_combinebackup/meson.build: endif - - pg_combinebackup = executable('pg_combinebackup', - pg_combinebackup_sources, -- dependencies: [frontend_code], -+ # XXX linking against libpq isn't good, but how was JSON working? -+ dependencies: [frontend_code, libpq], - kwargs: default_bin_args, - ) - bin_targets += pg_combinebackup - - ## src/bin/pg_verifybackup/Makefile ## -@@ src/bin/pg_verifybackup/Makefile: top_builddir = ../../.. - include $(top_builddir)/src/Makefile.global - - # We need libpq only because fe_utils does. --LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport) -+LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils -lpgcommon $(libpq_pgport) - - OBJS = \ - $(WIN32RES) \ - ## src/interfaces/libpq/Makefile ## @@ src/interfaces/libpq/Makefile: libpq-refs-stamp: $(shlib) ifneq ($(enable_coverage), yes) 9: c4d850a7c4 = 7: decc90579a WIP: Python OAuth provider implementation