1: 00976d4f75 ! 1: e2a0b48561 common/jsonapi: support FRONTEND clients @@ Commit message We can now partially revert b44669b2ca, now that json_errdetail() works correctly. + Co-authored-by: Daniel Gustafsson + ## src/bin/pg_verifybackup/t/005_bad_manifest.pl ## @@ src/bin/pg_verifybackup/t/005_bad_manifest.pl: use Test::More; my $tempdir = PostgreSQL::Test::Utils::tempdir; @@ src/common/jsonapi.c +#define appendStrValChar appendPQExpBufferChar +#define createStrVal createPQExpBuffer +#define resetStrVal resetPQExpBuffer ++#define destroyStrVal destroyPQExpBuffer + +#else /* !FRONTEND */ + @@ src/common/jsonapi.c +#define appendStrValChar appendStringInfoChar +#define createStrVal makeStringInfo +#define resetStrVal resetStringInfo ++#define destroyStrVal destroyStringInfo + +#endif + @@ src/common/jsonapi.c: makeJsonLexContextCstringLen(JsonLexContext *lex, char *js + static const JsonLexContext empty = {0}; + if (lex->flags & JSONLEX_FREE_STRVAL) - { -+#ifdef FRONTEND -+ destroyPQExpBuffer(lex->strval); -+#else - pfree(lex->strval->data); - pfree(lex->strval); -+#endif -+ } +- { +- pfree(lex->strval->data); +- pfree(lex->strval); +- } ++ destroyStrVal(lex->strval); ++ + if (lex->errormsg) -+ { -+#ifdef FRONTEND -+ destroyPQExpBuffer(lex->errormsg); -+#else -+ pfree(lex->errormsg->data); -+ pfree(lex->errormsg); -+#endif - } ++ destroyStrVal(lex->errormsg); ++ if (lex->flags & JSONLEX_FREE_STRUCT) pfree(lex); + else @@ src/common/parse_manifest.c: json_parse_manifest(JsonManifestParseContext *conte json_manifest_parse_failure(context, "manifest ended unexpectedly"); + ## src/common/stringinfo.c ## +@@ src/common/stringinfo.c: enlargeStringInfo(StringInfo str, int needed) + + str->maxlen = newlen; + } ++ ++void ++destroyStringInfo(StringInfo str) ++{ ++ pfree(str->data); ++ pfree(str); ++} + ## src/include/common/jsonapi.h ## @@ #ifndef JSONAPI_H @@ src/include/common/jsonapi.h: typedef struct JsonLexContext } JsonLexContext; typedef JsonParseErrorType (*json_struct_action) (void *state); + + ## src/include/lib/stringinfo.h ## +@@ src/include/lib/stringinfo.h: extern void appendBinaryStringInfoNT(StringInfo str, + */ + extern void enlargeStringInfo(StringInfo str, int needed); + ++ ++extern void destroyStringInfo(StringInfo str); + #endif /* STRINGINFO_H */ 2: d8b567dd55 ! 2: db625e1d01 Refactor SASL exchange to return tri-state status @@ src/interfaces/libpq/fe-auth-sasl.h: typedef struct pg_fe_sasl_mech + * SASL_CONTINUE: The output buffer is filled with a client response. + * Additional server challenge is expected + * SASL_COMPLETE: The SASL exchange has completed successfully. -+ * SASL_FAILED: The exchance has failed and the connection should be ++ * SASL_FAILED: The exchange has failed and the connection should be + * dropped. *-------- */ @@ src/interfaces/libpq/fe-auth.c: pg_SASL_continue(PGconn *conn, int payloadlen, b free(challenge); /* don't need the input anymore */ - if (final && !done) -+ if (final && !(status == SASL_FAILED || status == SASL_COMPLETE)) ++ if (final && status == SASL_CONTINUE) { if (outputlen != 0) free(output); 3: 83d78f598c ! 3: e4ad0260d5 Explicitly require password for SCRAM exchange @@ Commit message Discussion: https://postgr.es/m/d1b467a78e0e36ed85a09adf979d04cf124a9d4b.camel@vmware.com ## src/interfaces/libpq/fe-auth.c ## +@@ src/interfaces/libpq/fe-auth.c: pg_SASL_init(PGconn *conn, int payloadlen) + int initialresponselen; + const char *selected_mechanism; + PQExpBufferData mechanism_buf; +- char *password; ++ char *password = NULL; + SASLStatus status; + + initPQExpBuffer(&mechanism_buf); @@ src/interfaces/libpq/fe-auth.c: pg_SASL_init(PGconn *conn, int payloadlen) /* * Parse the list of SASL authentication mechanisms in the 4: 00c8073807 ! 4: 229f602d5c libpq: add OAUTHBEARER SASL mechanism @@ Commit message - figure out pgsocket/int difference on Windows - fix intermittent failure in the cleanup callback tests (race condition?) + - support require_auth - ...and more. + Co-authored-by: Daniel Gustafsson + ## configure ## @@ configure: with_uuid with_readline @@ src/interfaces/libpq/fe-auth-oauth-curl.c (new) + +/* + * Macros for getting and setting state for the connection's two cURL handles, -+ * so you don't have to write out the error handling every time. They assume -+ * that they're embedded in a function returning bool, however. ++ * so you don't have to write out the error handling every time. + */ + -+#define CHECK_MSETOPT(ACTX, OPT, VAL) \ ++#define CHECK_MSETOPT(ACTX, OPT, VAL, FAILACTION) \ + do { \ + struct async_ctx *_actx = (ACTX); \ + CURLMcode _setopterr = curl_multi_setopt(_actx->curlm, OPT, VAL); \ + if (_setopterr) { \ + actx_error(_actx, "failed to set %s on OAuth connection: %s",\ + #OPT, curl_multi_strerror(_setopterr)); \ -+ return false; \ ++ FAILACTION; \ + } \ + } while (0) + -+#define CHECK_SETOPT(ACTX, OPT, VAL) \ ++#define CHECK_SETOPT(ACTX, OPT, VAL, FAILACTION) \ + do { \ + struct async_ctx *_actx = (ACTX); \ + CURLcode _setopterr = curl_easy_setopt(_actx->curl, OPT, VAL); \ + if (_setopterr) { \ + actx_error(_actx, "failed to set %s on OAuth connection: %s",\ + #OPT, curl_easy_strerror(_setopterr)); \ -+ return false; \ ++ FAILACTION; \ + } \ + } while (0) + -+#define CHECK_GETINFO(ACTX, INFO, OUT) \ ++#define CHECK_GETINFO(ACTX, INFO, OUT, FAILACTION) \ + do { \ + struct async_ctx *_actx = (ACTX); \ + CURLcode _getinfoerr = curl_easy_getinfo(_actx->curl, INFO, OUT); \ + if (_getinfoerr) { \ + actx_error(_actx, "failed to get %s from OAuth response: %s",\ + #INFO, curl_easy_strerror(_getinfoerr)); \ -+ return false; \ ++ FAILACTION; \ + } \ + } while (0) + @@ src/interfaces/libpq/fe-auth-oauth-curl.c (new) + + while (field->name) + { -+ if (!strcmp(name, field->name)) ++ if (strcmp(name, field->name) == 0) + { + ctx->active = field; + break; @@ src/interfaces/libpq/fe-auth-oauth-curl.c (new) + bool success = false; + + /* Make sure the server thinks it's given us JSON. */ -+ CHECK_GETINFO(actx, CURLINFO_CONTENT_TYPE, &content_type); ++ CHECK_GETINFO(actx, CURLINFO_CONTENT_TYPE, &content_type, return false); + + if (!content_type) + { @@ src/interfaces/libpq/fe-auth-oauth-curl.c (new) +static int +parse_interval(const char *interval_str) +{ -+ float parsed; ++ double parsed; + int cnt; + + /* + * The JSON lexer has already validated the number, which is stricter than + * the %f format, so we should be good to use sscanf(). + */ -+ cnt = sscanf(interval_str, "%f", &parsed); ++ cnt = sscanf(interval_str, "%lf", &parsed); + + if (cnt != 1) + { @@ src/interfaces/libpq/fe-auth-oauth-curl.c (new) + return 1; /* don't fall through in release builds */ + } + -+ parsed = ceilf(parsed); ++ parsed = ceil(parsed); + + if (parsed < 1) + return 1; /* TODO this slows down the tests @@ src/interfaces/libpq/fe-auth-oauth-curl.c (new) +static bool +setup_curl_handles(struct async_ctx *actx) +{ ++ curl_version_info_data *curl_info; ++ + /* + * Create our multi handle. This encapsulates the entire conversation with + * cURL for this connection. @@ src/interfaces/libpq/fe-auth-oauth-curl.c (new) + } + + /* ++ * Extract information about the libcurl we are linked against. ++ */ ++ curl_info = curl_version_info(CURLVERSION_NOW); ++ ++ /* + * The multi handle tells us what to wait on using two callbacks. These + * will manipulate actx->mux as needed. + */ -+ CHECK_MSETOPT(actx, CURLMOPT_SOCKETFUNCTION, register_socket); -+ CHECK_MSETOPT(actx, CURLMOPT_SOCKETDATA, actx); -+ CHECK_MSETOPT(actx, CURLMOPT_TIMERFUNCTION, register_timer); -+ CHECK_MSETOPT(actx, CURLMOPT_TIMERDATA, actx); ++ CHECK_MSETOPT(actx, CURLMOPT_SOCKETFUNCTION, register_socket, return false); ++ CHECK_MSETOPT(actx, CURLMOPT_SOCKETDATA, actx, return false); ++ CHECK_MSETOPT(actx, CURLMOPT_TIMERFUNCTION, register_timer, return false); ++ CHECK_MSETOPT(actx, CURLMOPT_TIMERDATA, actx, return false); + + /* + * Set up an easy handle. All of our requests are made serially, so we @@ src/interfaces/libpq/fe-auth-oauth-curl.c (new) + * Multi-threaded applications must set CURLOPT_NOSIGNAL. This requires us + * to handle the possibility of SIGPIPE ourselves. + * -+ * TODO: This disables DNS resolution timeouts unless libcurl has been -+ * compiled against alternative resolution support. We should check that. -+ * + * TODO: handle SIGPIPE via pq_block_sigpipe(), or via a + * CURLOPT_SOCKOPTFUNCTION maybe... + */ -+ CHECK_SETOPT(actx, CURLOPT_NOSIGNAL, 1L); ++ CHECK_SETOPT(actx, CURLOPT_NOSIGNAL, 1L, return false); ++ if (!curl_info->ares_num) ++ { ++ /* No alternative resolver, TODO: warn about timeouts */ ++ } + + /* TODO investigate using conn->Pfdebug and CURLOPT_DEBUGFUNCTION here */ -+ CHECK_SETOPT(actx, CURLOPT_VERBOSE, 1L); -+ CHECK_SETOPT(actx, CURLOPT_ERRORBUFFER, actx->curl_err); -+ -+ /* TODO */ -+ CHECK_SETOPT(actx, CURLOPT_WRITEDATA, stderr); ++ CHECK_SETOPT(actx, CURLOPT_VERBOSE, 1L, return false); ++ CHECK_SETOPT(actx, CURLOPT_ERRORBUFFER, actx->curl_err, return false); + + /* + * Only HTTP[S] is allowed. TODO: disallow HTTP without user opt-in + */ -+ CHECK_SETOPT(actx, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS); ++ CHECK_SETOPT(actx, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS, return false); + + /* + * Suppress the Accept header to make our request as minimal as possible. @@ src/interfaces/libpq/fe-auth-oauth-curl.c (new) + * what comes back anyway.) + */ + actx->headers = curl_slist_append(actx->headers, "Accept:"); /* TODO: check result */ -+ CHECK_SETOPT(actx, CURLOPT_HTTPHEADER, actx->headers); ++ CHECK_SETOPT(actx, CURLOPT_HTTPHEADER, actx->headers, return false); + + return true; +} @@ src/interfaces/libpq/fe-auth-oauth-curl.c (new) + int running; + + resetPQExpBuffer(&actx->work_data); -+ CHECK_SETOPT(actx, CURLOPT_WRITEFUNCTION, append_data); -+ CHECK_SETOPT(actx, CURLOPT_WRITEDATA, &actx->work_data); ++ CHECK_SETOPT(actx, CURLOPT_WRITEFUNCTION, append_data, return false); ++ CHECK_SETOPT(actx, CURLOPT_WRITEDATA, &actx->work_data, return false); + + err = curl_multi_add_handle(actx->curlm, actx->curl); + if (err) @@ src/interfaces/libpq/fe-auth-oauth-curl.c (new) +static bool +start_discovery(struct async_ctx *actx, const char *discovery_uri) +{ -+ CHECK_SETOPT(actx, CURLOPT_HTTPGET, 1L); -+ CHECK_SETOPT(actx, CURLOPT_URL, discovery_uri); ++ CHECK_SETOPT(actx, CURLOPT_HTTPGET, 1L, return false); ++ CHECK_SETOPT(actx, CURLOPT_URL, discovery_uri, return false); + + return start_request(actx); +} @@ src/interfaces/libpq/fe-auth-oauth-curl.c (new) + * validation into question), or non-authoritative responses, or any other + * complications. + */ -+ CHECK_GETINFO(actx, CURLINFO_RESPONSE_CODE, &response_code); ++ CHECK_GETINFO(actx, CURLINFO_RESPONSE_CODE, &response_code, return false); + + if (response_code != 200) + { @@ src/interfaces/libpq/fe-auth-oauth-curl.c (new) + * Per Section 3, the default is ["authorization_code", "implicit"]. + */ + struct curl_slist *temp = actx->provider.grant_types_supported; -+ bool oom = false; + + temp = curl_slist_append(temp, "authorization_code"); -+ if (!temp) -+ oom = true; ++ if (temp) ++ { ++ temp = curl_slist_append(temp, "implicit"); ++ } + -+ temp = curl_slist_append(temp, "implicit"); + if (!temp) -+ oom = true; -+ -+ if (oom) + { + actx_error(actx, "out of memory"); + return false; @@ src/interfaces/libpq/fe-auth-oauth-curl.c (new) + /* TODO check for broken buffer */ + + /* Make our request. */ -+ CHECK_SETOPT(actx, CURLOPT_URL, device_authz_uri); -+ CHECK_SETOPT(actx, CURLOPT_COPYPOSTFIELDS, work_buffer->data); ++ CHECK_SETOPT(actx, CURLOPT_URL, device_authz_uri, return false); ++ CHECK_SETOPT(actx, CURLOPT_COPYPOSTFIELDS, work_buffer->data, return false); + + if (conn->oauth_client_secret) + { @@ src/interfaces/libpq/fe-auth-oauth-curl.c (new) + * + * TODO: should we omit client_id from the body in this case? + */ -+ CHECK_SETOPT(actx, CURLOPT_HTTPAUTH, CURLAUTH_BASIC); -+ CHECK_SETOPT(actx, CURLOPT_USERNAME, conn->oauth_client_id); -+ CHECK_SETOPT(actx, CURLOPT_PASSWORD, conn->oauth_client_secret); ++ CHECK_SETOPT(actx, CURLOPT_HTTPAUTH, CURLAUTH_BASIC, return false); ++ CHECK_SETOPT(actx, CURLOPT_USERNAME, conn->oauth_client_id, return false); ++ CHECK_SETOPT(actx, CURLOPT_PASSWORD, conn->oauth_client_secret, return false); + } + else -+ CHECK_SETOPT(actx, CURLOPT_HTTPAUTH, CURLAUTH_NONE); ++ CHECK_SETOPT(actx, CURLOPT_HTTPAUTH, CURLAUTH_NONE, return false); + + return start_request(actx); +} @@ src/interfaces/libpq/fe-auth-oauth-curl.c (new) +{ + long response_code; + -+ CHECK_GETINFO(actx, CURLINFO_RESPONSE_CODE, &response_code); ++ CHECK_GETINFO(actx, CURLINFO_RESPONSE_CODE, &response_code, return false); + + /* + * The device authorization endpoint uses the same error response as the @@ src/interfaces/libpq/fe-auth-oauth-curl.c (new) + /* TODO check for broken buffer */ + + /* Make our request. */ -+ CHECK_SETOPT(actx, CURLOPT_URL, token_uri); -+ CHECK_SETOPT(actx, CURLOPT_COPYPOSTFIELDS, work_buffer->data); ++ CHECK_SETOPT(actx, CURLOPT_URL, token_uri, return false); ++ CHECK_SETOPT(actx, CURLOPT_COPYPOSTFIELDS, work_buffer->data, return false); + + if (conn->oauth_client_secret) + { @@ src/interfaces/libpq/fe-auth-oauth-curl.c (new) + * + * TODO: should we omit client_id from the body in this case? + */ -+ CHECK_SETOPT(actx, CURLOPT_HTTPAUTH, CURLAUTH_BASIC); -+ CHECK_SETOPT(actx, CURLOPT_USERNAME, conn->oauth_client_id); -+ CHECK_SETOPT(actx, CURLOPT_PASSWORD, conn->oauth_client_secret); ++ CHECK_SETOPT(actx, CURLOPT_HTTPAUTH, CURLAUTH_BASIC, return false); ++ CHECK_SETOPT(actx, CURLOPT_USERNAME, conn->oauth_client_id, return false); ++ CHECK_SETOPT(actx, CURLOPT_PASSWORD, conn->oauth_client_secret, return false); + } + else -+ CHECK_SETOPT(actx, CURLOPT_HTTPAUTH, CURLAUTH_NONE); ++ CHECK_SETOPT(actx, CURLOPT_HTTPAUTH, CURLAUTH_NONE, return false); + + resetPQExpBuffer(work_buffer); -+ CHECK_SETOPT(actx, CURLOPT_WRITEFUNCTION, append_data); -+ CHECK_SETOPT(actx, CURLOPT_WRITEDATA, work_buffer); ++ CHECK_SETOPT(actx, CURLOPT_WRITEFUNCTION, append_data, return false); ++ CHECK_SETOPT(actx, CURLOPT_WRITEDATA, work_buffer, return false); + + return start_request(actx); +} @@ src/interfaces/libpq/fe-auth-oauth-curl.c (new) +{ + long response_code; + -+ CHECK_GETINFO(actx, CURLINFO_RESPONSE_CODE, &response_code); ++ CHECK_GETINFO(actx, CURLINFO_RESPONSE_CODE, &response_code, return false); + + /* + * Per RFC 6749, Section 5, a successful response uses 200 OK. An error @@ src/interfaces/libpq/fe-auth-oauth-curl.c (new) + * A slow_down error requires us to permanently increase our + * retry interval by five seconds. RFC 8628, Sec. 3.5. + */ -+ if (!strcmp(err->error, "slow_down")) ++ if (strcmp(err->error, "slow_down") == 0) + { + actx->authz.interval += 5; /* TODO check for overflow? */ + } @@ src/interfaces/libpq/fe-auth-oauth.c (new) + * error. + */ + Assert(sasl_mechanism != NULL); -+ Assert(!strcmp(sasl_mechanism, OAUTHBEARER_NAME)); ++ Assert(strcmp(sasl_mechanism, OAUTHBEARER_NAME) == 0); + + state = calloc(1, sizeof(*state)); + if (!state) @@ src/interfaces/libpq/fe-auth-oauth.c (new) + + if (ctx->nested == 1) + { -+ if (!strcmp(name, ERROR_STATUS_FIELD)) ++ if (strcmp(name, ERROR_STATUS_FIELD) == 0) + { + ctx->target_field_name = ERROR_STATUS_FIELD; + ctx->target_field = &ctx->status; + } -+ else if (!strcmp(name, ERROR_SCOPE_FIELD)) ++ else if (strcmp(name, ERROR_SCOPE_FIELD) == 0) + { + ctx->target_field_name = ERROR_SCOPE_FIELD; + ctx->target_field = &ctx->scope; + } -+ else if (!strcmp(name, ERROR_OPENID_CONFIGURATION_FIELD)) ++ else if (strcmp(name, ERROR_OPENID_CONFIGURATION_FIELD) == 0) + { + ctx->target_field_name = ERROR_OPENID_CONFIGURATION_FIELD; + ctx->target_field = &ctx->discovery_uri; @@ src/interfaces/libpq/fe-auth-oauth.c (new) + if (strlen(msg) != msglen) + { + appendPQExpBufferStr(&conn->errorMessage, -+ libpq_gettext("server's error message contained an embedded NULL")); ++ libpq_gettext("server's error message contained an embedded NULL, and was discarded")); + return false; + } + @@ src/interfaces/libpq/fe-auth-oauth.c (new) + return false; + } + -+ if (!strcmp(ctx.status, "invalid_token")) ++ if (strcmp(ctx.status, "invalid_token") == 0) + { + /* + * invalid_token is the only error code we'll automatically retry for, @@ src/interfaces/libpq/fe-auth-sasl.h: typedef struct pg_fe_sasl_mech + * generated. The mechanism is responsible for setting up + * conn->async_auth appropriately before returning. * SASL_COMPLETE: The SASL exchange has completed successfully. -- * SASL_FAILED: The exchance has failed and the connection should be -+ * SASL_FAILED: The exchange has failed and the connection should be + * SASL_FAILED: The exchange has failed and the connection should be * dropped. *-------- */ @@ src/interfaces/libpq/fe-auth.c: pg_SSPI_startup(PGconn *conn, int use_negotiate, { char *initialresponse = NULL; int initialresponselen; - const char *selected_mechanism; - PQExpBufferData mechanism_buf; -- char *password; -+ char *password = NULL; - SASLStatus status; - - initPQExpBuffer(&mechanism_buf); @@ src/interfaces/libpq/fe-auth.c: pg_SASL_init(PGconn *conn, int payloadlen) goto error; } @@ src/interfaces/libpq/fe-auth.c: pg_SASL_continue(PGconn *conn, int payloadlen, b + return STATUS_OK; + } + - if (final && !(status == SASL_FAILED || status == SASL_COMPLETE)) + if (final && status == SASL_CONTINUE) { if (outputlen != 0) @@ src/interfaces/libpq/fe-auth.c: check_expected_areq(AuthRequest areq, PGconn *conn) 5: 29d7e3cbed ! 5: 5c5a83e44e backend: add OAUTHBEARER SASL mechanism @@ Commit message deal with multi-issuer setups - ...and more. + Co-authored-by: Daniel Gustafsson + ## src/backend/libpq/Makefile ## @@ src/backend/libpq/Makefile: include $(top_builddir)/src/Makefile.global # be-fsstubs is here for historical reasons, probably belongs elsewhere @@ src/backend/libpq/auth-oauth.c (new) + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + errmsg("malformed OAUTHBEARER message"), -+ errdetail("Comma expected, but found character %s.", ++ errdetail("Comma expected, but found character \"%s\".", + sanitize_char(*p)))); + p++; + break; 6: 0661817808 = 6: 7a42365d62 Introduce OAuth validator libraries 7: 45755e8461 ! 7: 9c46ea6cf9 Add pytest suite for OAuth @@ Commit message TODOs: - The --tap-stream option to pytest-tap is slightly broken during test failures (it suppresses error information), which impedes debugging. - - Unsurprisingly, Windows builds fail on the Linux-/BSD-specific backend - changes. 32-bit builds on Ubuntu fail during testing as well. - pyca/cryptography is pinned at an old version. Since we use it for testing and not security, this isn't a critical problem yet, but it's not ideal. Newer versions require a Rust compiler to build, and while @@ Commit message with the Rust pieces bypassed, compilation on FreeBSD takes a while. - The with_oauth test skip logic should probably be integrated into the Makefile side as well... + - See if 32-bit tests can be enabled with a 32-bit Python. ## .cirrus.tasks.yml ## @@ .cirrus.tasks.yml: env: @@ .cirrus.tasks.yml: task: matrix: - name: Linux - Debian Bullseye - Autoconf +@@ .cirrus.tasks.yml: task: + + # Also build & test in a 32bit build - it's gotten rare to test that + # locally. ++ # XXX 32-bit Python tests are currently disabled, as the system's 64-bit ++ # Python modules can't link against libpq. + configure_32_script: | + su postgres <<-EOF + export CC='ccache gcc -m32' +@@ .cirrus.tasks.yml: task: + -Dllvm=disabled \ + --pkg-config-path /usr/lib/i386-linux-gnu/pkgconfig/ \ + -DPERL=perl5.32-i386-linux-gnu \ +- -DPG_TEST_EXTRA="$PG_TEST_EXTRA" \ ++ -DPG_TEST_EXTRA="${PG_TEST_EXTRA//"python"}" \ + build-32 + EOF + @@ .cirrus.tasks.yml: task: folder: $CCACHE_DIR @@ meson.build: foreach test_dir : tests + env.set(name, value) + endforeach + -+ reqs = files(t['requirements']) -+ test('install_' + venv_name, -+ python, -+ args: [ make_venv, '--requirements', reqs, venv_path ], -+ env: env, -+ priority: setup_tests_priority - 1, # must run after tmp_install -+ is_parallel: false, -+ suite: ['setup'], -+ timeout: 60, # 30s is too short for the cryptography package compile -+ ) ++ if get_option('PG_TEST_EXTRA').contains('python') ++ reqs = files(t['requirements']) ++ test('install_' + venv_name, ++ python, ++ args: [ make_venv, '--requirements', reqs, venv_path ], ++ env: env, ++ priority: setup_tests_priority - 1, # must run after tmp_install ++ is_parallel: false, ++ suite: ['setup'], ++ timeout: 60, # 30s is too short for the cryptography package compile ++ ) ++ endif + + test_group = test_dir['name'] + test_output = test_result_dir / test_group / kind @@ meson.build: foreach test_dir : tests + 'env': env, + } + -+ pytest = venv_path / 'bin' / 'py.test' ++ if fs.is_dir(venv_path / 'Scripts') ++ # Windows virtualenv layout ++ pytest = venv_path / 'Scripts' / 'py.test' ++ else ++ pytest = venv_path / 'bin' / 'py.test' ++ endif ++ + test_command = [ + pytest, + # Avoid running these tests against an existing database. @@ meson.build: foreach test_dir : tests + pyt_p = fs.stem(pyt_p) + endif + ++ testwrap_pytest = testwrap_base + [ ++ '--testgroup', test_group, ++ '--testname', pyt_p, ++ ] ++ if not get_option('PG_TEST_EXTRA').contains('python') ++ testwrap_pytest += ['--skip', '"python" tests not enabled in PG_TEST_EXTRA'] ++ endif ++ + test(test_group / pyt_p, + python, + kwargs: test_kwargs, -+ args: testwrap_base + [ -+ '--testgroup', test_group, -+ '--testname', pyt_p, ++ args: testwrap_pytest + [ + '--', test_command, + test_dir['sd'] / pyt, + ], @@ src/test/python/client/test_oauth.py (new) + +if platform.system() == "Darwin": + libpq = ctypes.cdll.LoadLibrary("libpq.5.dylib") ++elif platform.system() == "Windows": ++ pass # TODO +else: + libpq = ctypes.cdll.LoadLibrary("libpq.so.5") + @@ src/test/python/conftest.py (new) + Automatically skips the whole suite if PG_TEST_EXTRA doesn't contain + 'python'. pytestmark doesn't seem to work in a top-level conftest.py, so + I've made this an autoused fixture instead. -+ -+ TODO: there are tests here that are probably safe, but until I do a full -+ analysis on which are and which are not, I've made the entire thing opt-in. + """ + extra_tests = os.getenv("PG_TEST_EXTRA", "").split() + if "python" not in extra_tests: @@ src/test/python/pq3.py (new) +import getpass +import io +import os ++import platform +import ssl +import sys +import textwrap @@ src/test/python/pq3.py (new) + try: + return os.environ["PGUSER"] + except KeyError: ++ if platform.system() == "Windows": ++ # libpq defaults to GetUserName() on Windows. ++ return os.getlogin() + return getpass.getuser() + + @@ src/test/python/server/meson.build (new) @@ +# Copyright (c) 2024, PostgreSQL Global Development Group + -+if not oauth.found() -+ subdir_done() -+endif -+ +oauthtest_sources = files( + 'oauthtest.c', +) @@ src/test/python/server/test_oauth.py (new) +import json +import os +import pathlib ++import platform +import secrets +import shlex +import shutil @@ src/test/python/server/test_oauth.py (new) + scope = "openid " + id + + ctx = Context() -+ hba_lines = ( ++ hba_lines = [ + f'host {ctx.dbname} {ctx.map_user} samehost oauth issuer="{ctx.issuer}" scope="{ctx.scope}" map=oauth\n', + f'host {ctx.dbname} {ctx.authz_user} samehost oauth issuer="{ctx.issuer}" scope="{ctx.scope}" trust_validator_authz=1\n', + f'host {ctx.dbname} all samehost oauth issuer="{ctx.issuer}" scope="{ctx.scope}"\n', -+ ) -+ ident_lines = (r"oauth /^(.*)@example\.com$ \1",) ++ ] ++ ident_lines = [r"oauth /^(.*)@example\.com$ \1"] ++ ++ if platform.system() == "Windows": ++ # XXX why is 'samehost' not behaving as expected on Windows? ++ for l in list(hba_lines): ++ hba_lines.append(l.replace("samehost", "::1/128")) + + host, port = postgres_instance + conn = psycopg2.connect(host=host, port=port) @@ src/test/python/test_pq3.py (new) +import contextlib +import getpass +import io ++import platform +import struct +import sys + @@ src/test/python/test_pq3.py (new) + [ + ("PGHOST", pq3.pghost, "localhost"), + ("PGPORT", pq3.pgport, 5432), -+ ("PGUSER", pq3.pguser, getpass.getuser()), ++ ( ++ "PGUSER", ++ pq3.pguser, ++ os.getlogin() if platform.system() == "Windows" else getpass.getuser(), ++ ), + ("PGDATABASE", pq3.pgdatabase, "postgres"), + ], +) @@ src/tools/make_venv (new) +import argparse +import subprocess +import os ++import platform +import sys + +parser = argparse.ArgumentParser() @@ src/tools/make_venv (new) + +# Update pip next. This helps avoid old pip bugs; the version inside system +# Pythons tends to be pretty out of date. -+pip = os.path.join(args.venv_path, 'bin', 'pip') -+run(pip, 'install', '-U', 'pip') ++bindir = 'Scripts' if platform.system() == 'Windows' else 'bin' ++python = os.path.join(args.venv_path, bindir, 'python3') ++run(python, '-m', 'pip', 'install', '-U', 'pip') + +# Finally, install the test's requirements. We need pytest and pytest-tap, no +# matter what the test needs. ++pip = os.path.join(args.venv_path, bindir, 'pip') +run(pip, 'install', 'pytest', 'pytest-tap') +if args.requirements: + run(pip, 'install', '-r', args.requirements) 8: 0f9f884856 ! 8: 8ad4ce3068 XXX temporary patches to build and test @@ src/bin/pg_verifybackup/Makefile: top_builddir = ../../.. ## src/interfaces/libpq/Makefile ## @@ src/interfaces/libpq/Makefile: libpq-refs-stamp: $(shlib) ifneq ($(enable_coverage), yes) - ifeq (,$(filter aix solaris,$(PORTNAME))) + ifeq (,$(filter solaris,$(PORTNAME))) @if nm -A -u $< 2>/dev/null | grep -v -e __cxa_atexit -e __tsan_func_exit | grep exit; then \ - echo 'libpq must not be calling any function which invokes exit'; exit 1; \ + echo 'libpq must not be calling any function which invokes exit'; \ 9: de8f81bd7d ! 9: 5630465578 WIP: Python OAuth provider implementation @@ src/test/modules/oauth_validator/t/oauth_server.py (new) +import json +import os +import sys -+from typing import TypeAlias + + +class OAuthHandler(http.server.BaseHTTPRequestHandler): -+ JsonObject: TypeAlias = dict[str, object] ++ JsonObject = dict[str, object] # TypeAlias is not available until 3.10 + + def do_GET(self): + if self.path == "/.well-known/openid-configuration":