From 5e958ce1abb587eb0794fce7533259e66862dfc7 Mon Sep 17 00:00:00 2001 From: CommanderKeynes Date: Sun, 23 Mar 2025 22:03:45 -0500 Subject: [PATCH] Add http connection paramater lookup functionality Adds 3 exported functions to the libpq-oauth shared object file that are used in the httpServiceLookup function in fe-connect.c to read connection parameters from an HTTP web server. The goal here is to provide the ability for managed service operators to define a single source of truth for connection details. The implementation reflects the currently existing implementation of libpq ldap connection parameter lookup. This enables a form of built-in libpq service discovery format. This would allow administrators to add, remove, and change hosts in multi host connection strings without coordinating with every end user who may hardcode their connection strings in a lot of different places. While the libpq-ldap functionality accomplished the above goal setting up LDAP infrastructure is a lot less accessible to many administrators than setting up an HTTP web server. In many cases it would be sufficient to "just throw some files on an s3 bucket". The current state of this patch is rough and I am happy to accept any feedback. Some issues: 1. Bundling this functionality in with libpq-oauth.so seems odd. It would probably make more sense to rename libpq-oauth.so to libpq-oauth.so to libpq-libcurl.so or create an entirely new .so file for this logic. 2. Tests may not sit in the correct place. 3. The code implemented in libpq-oauth.so is implemeneted in an file open/read/close like interface borrowed from libcurl's docs [4]. This may not be the best production ready approach for this implementation and happy to get this updatedd if need be. Despite these shortcomings this approach may be a more natural alternative to previous attempts [2, 3] at allowing administrators to mix read-only/read-write nodes into overloaded A records. [0]: https://www.postgresql.org/docs/current/libpq-ldap.html [1]: https://www.postgresql.org/message-id/flat/CAKK5BkFOFGfKJNbTuYBvE0PfpHmW8iZEmdNogaCYqjAOhtNgDg@mail.gmail.com [2]: https://www.postgresql.org/message-id/flat/CAKK5BkESSc69sp2TiTWHvvOHCUey0rDWXSrR9pinyRqyfamUYg@mail.gmail.com [3]: https://www.postgresql.org/message-id/AM9PR09MB49008B02CDF003054D5D4E00977DA@AM9PR09MB4900.eurprd09.prod.outlook.com [4]: https://curl.se/libcurl/c/fopen.html --- doc/src/sgml/libpq.sgml | 40 +++ src/interfaces/libpq-oauth/Makefile | 1 + src/interfaces/libpq-oauth/exports.txt | 7 +- src/interfaces/libpq-oauth/http-service.c | 280 ++++++++++++++++++ src/interfaces/libpq-oauth/http-service.h | 52 ++++ src/interfaces/libpq-oauth/meson.build | 1 + src/interfaces/libpq/fe-connect.c | 269 +++++++++++++++++ src/test/modules/Makefile | 1 + src/test/modules/http_pg_service/Makefile | 37 +++ src/test/modules/http_pg_service/meson.build | 17 ++ .../t/001_http_service_file.pl | 200 +++++++++++++ .../http_pg_service/t/PgHttpService/Server.pm | 141 +++++++++ .../http_pg_service/t/http_service_server.py | 73 +++++ 13 files changed, 1117 insertions(+), 2 deletions(-) create mode 100644 src/interfaces/libpq-oauth/http-service.c create mode 100644 src/interfaces/libpq-oauth/http-service.h create mode 100644 src/test/modules/http_pg_service/Makefile create mode 100644 src/test/modules/http_pg_service/meson.build create mode 100644 src/test/modules/http_pg_service/t/001_http_service_file.pl create mode 100644 src/test/modules/http_pg_service/t/PgHttpService/Server.pm create mode 100644 src/test/modules/http_pg_service/t/http_service_server.py diff --git a/doc/src/sgml/libpq.sgml b/doc/src/sgml/libpq.sgml index 6db823808fc..f1a100c0b65 100644 --- a/doc/src/sgml/libpq.sgml +++ b/doc/src/sgml/libpq.sgml @@ -9721,6 +9721,46 @@ ldap://ldap.acme.com/cn=dbserver,cn=hosts?pgconnectinfo?base?(objectclass=*) + + HTTP Lookup of Connection Parameters + + + HTTPconnection parameter lookup + + + + If libpq has been compiled with libcurl support (option + ) for configure + it is possible to retrieve connection options like host + or dbname via HTTP from a central server. + The advantage is that if the connection parameters for a database change, + the connection information doesn't have to be updated on all client machines. + + + + LDAP connection parameter lookup uses the connection service file + pg_service.conf (see ). A line in a + pg_service.conf stanza that starts with + http:// will be recognized as an HTTP URL and an + HTTP query will be performed. The result must be a list of + keyword = value pairs which will be used to set + connection options. The URL must be of the form + +http://[hostname[:port]]/path + + + + + Processing of pg_service.conf is terminated after + a successful HTTP lookup, but is continued if the HTTP server cannot + be contacted. This is to provide a fallback with further HTTP URL + lines that point to different HTTP servers, classical keyword + = value pairs, or default connection options. If you would + rather get an error message in this case, add a syntactically incorrect + line after the LDAP URL. + + SSL Support diff --git a/src/interfaces/libpq-oauth/Makefile b/src/interfaces/libpq-oauth/Makefile index 11e1a3cf528..6d4a1f240cc 100644 --- a/src/interfaces/libpq-oauth/Makefile +++ b/src/interfaces/libpq-oauth/Makefile @@ -42,6 +42,7 @@ OBJS_STATIC = oauth-curl.o OBJS_SHLIB = \ oauth-curl_shlib.o \ oauth-utils.o \ + http-service.o \ oauth-utils.o: override CPPFLAGS += $(CPPFLAGS_SHLIB) diff --git a/src/interfaces/libpq-oauth/exports.txt b/src/interfaces/libpq-oauth/exports.txt index 7bc12b860d7..553eb391458 100644 --- a/src/interfaces/libpq-oauth/exports.txt +++ b/src/interfaces/libpq-oauth/exports.txt @@ -1,3 +1,6 @@ # src/interfaces/libpq-oauth/exports.txt -libpq_oauth_init 1 -pg_start_oauthbearer 2 +libpq_oauth_init 1 +pg_start_oauthbearer 2 +url_fopen 3 +url_fgets 4 +url_fclose 5 diff --git a/src/interfaces/libpq-oauth/http-service.c b/src/interfaces/libpq-oauth/http-service.c new file mode 100644 index 00000000000..7faf778d075 --- /dev/null +++ b/src/interfaces/libpq-oauth/http-service.c @@ -0,0 +1,280 @@ +#include +#include +#include +#include + +#include "postgres_fe.h" +#include "pqexpbuffer.h" +#include +#include "oauth-utils.h" +#include "http-service.h" + +/* we use a global one for convenience */ +static CURLM *multi_handle; + +/* curl calls this routine to get more data */ +static size_t write_callback(char *buffer, + size_t size, + size_t nitems, + void *userp) +{ + char *newbuff; + size_t rembuff; + + URL_FILE *url = (URL_FILE *)userp; + size *= nitems; + + rembuff = url->buffer_len - url->buffer_pos; /* remaining space in buffer */ + + if(size > rembuff) { + /* not enough space in buffer */ + newbuff = realloc(url->buffer, url->buffer_len + (size - rembuff)); + if(!newbuff) { + fprintf(stderr, "callback buffer grow failed\n"); + size = rembuff; + } + else { + /* realloc succeeded increase buffer size*/ + url->buffer_len += size - rembuff; + url->buffer = newbuff; + } + } + + memcpy(&url->buffer[url->buffer_pos], buffer, size); + url->buffer_pos += size; + + return size; +} + +/* use to attempt to fill the read buffer up to requested number of bytes */ +static int fill_buffer(URL_FILE *file, size_t want) +{ + fd_set fdread; + fd_set fdwrite; + fd_set fdexcep; + struct timeval timeout; + int rc; + CURLMcode mc; /* curl_multi_fdset() return code */ + + /* only attempt to fill buffer if transactions still running and buffer + * does not exceed required size already + */ + if((!file->still_running) || (file->buffer_pos > want)) + return 0; + + /* attempt to fill buffer */ + do { + int maxfd = -1; + long curl_timeo = -1; + + FD_ZERO(&fdread); + FD_ZERO(&fdwrite); + FD_ZERO(&fdexcep); + + /* set a suitable timeout to fail on */ + timeout.tv_sec = 60; /* 1 minute */ + timeout.tv_usec = 0; + + curl_multi_timeout(multi_handle, &curl_timeo); + if(curl_timeo >= 0) { + timeout.tv_sec = curl_timeo / 1000; + if(timeout.tv_sec > 1) + timeout.tv_sec = 1; + else + timeout.tv_usec = (curl_timeo % 1000) * 1000; + } + + /* get file descriptors from the transfers */ + mc = curl_multi_fdset(multi_handle, &fdread, &fdwrite, &fdexcep, &maxfd); + + if(mc != CURLM_OK) { + fprintf(stderr, "curl_multi_fdset() failed, code %d.\n", mc); + break; + } + + /* On success the value of maxfd is guaranteed to be >= -1. We call + select(maxfd + 1, ...); specially in case of (maxfd == -1) there are + no fds ready yet so we call select(0, ...) --or Sleep() on Windows-- + to sleep 100ms, which is the minimum suggested value in the + curl_multi_fdset() doc. */ + + if(maxfd == -1) { +#ifdef _WIN32 + Sleep(100); + rc = 0; +#else + /* Portable sleep for platforms other than Windows. */ + struct timeval wait = { 0, 100 * 1000 }; /* 100ms */ + rc = select(0, NULL, NULL, NULL, &wait); +#endif + } + else { + /* Note that on some platforms 'timeout' may be modified by select(). + If you need access to the original value save a copy beforehand. */ + rc = select(maxfd + 1, &fdread, &fdwrite, &fdexcep, &timeout); + } + + switch(rc) { + case -1: + /* select error */ + break; + + case 0: + default: + /* timeout or readable/writable sockets */ + curl_multi_perform(multi_handle, &file->still_running); + break; + } + } while(file->still_running && (file->buffer_pos < want)); + return 1; +} + +/* use to remove want bytes from the front of a files buffer */ +static int use_buffer(URL_FILE *file, size_t want) +{ + /* sort out buffer */ + if(file->buffer_pos <= want) { + /* ditch buffer - write will recreate */ + free(file->buffer); + file->buffer = NULL; + file->buffer_pos = 0; + file->buffer_len = 0; + } + else { + /* move rest down make it available for later */ + memmove(file->buffer, + &file->buffer[want], + (file->buffer_pos - want)); + + file->buffer_pos -= want; + } + return 0; +} + +URL_FILE *url_fopen(const char *url, const char *operation) +{ + /* this code could check for URLs or types in the 'url' and + basically use the real fopen() for standard files */ + + URL_FILE *file; + (void)operation; + + file = calloc(1, sizeof(URL_FILE)); + if(!file) + return NULL; + + file->handle.file = fopen(url, operation); + if(file->handle.file) + file->type = CFTYPE_FILE; /* marked as URL */ + + else { + file->type = CFTYPE_CURL; /* marked as URL */ + file->handle.curl = curl_easy_init(); + + curl_easy_setopt(file->handle.curl, CURLOPT_URL, url); + curl_easy_setopt(file->handle.curl, CURLOPT_WRITEDATA, file); + curl_easy_setopt(file->handle.curl, CURLOPT_VERBOSE, 0L); + curl_easy_setopt(file->handle.curl, CURLOPT_WRITEFUNCTION, write_callback); + + if(!multi_handle) + multi_handle = curl_multi_init(); + + curl_multi_add_handle(multi_handle, file->handle.curl); + + /* lets start the fetch */ + curl_multi_perform(multi_handle, &file->still_running); + + if((file->buffer_pos == 0) && (!file->still_running)) { + /* if still_running is 0 now, we should return NULL */ + + /* make sure the easy handle is not in the multi handle anymore */ + curl_multi_remove_handle(multi_handle, file->handle.curl); + + /* cleanup */ + curl_easy_cleanup(file->handle.curl); + + free(file); + + file = NULL; + } + } + return file; +} + +int url_fclose(URL_FILE *file) +{ + int ret = 0;/* default is good return */ + + switch(file->type) { + case CFTYPE_FILE: + ret = fclose(file->handle.file); /* passthrough */ + break; + + case CFTYPE_CURL: + /* make sure the easy handle is not in the multi handle anymore */ + curl_multi_remove_handle(multi_handle, file->handle.curl); + + /* cleanup */ + curl_easy_cleanup(file->handle.curl); + break; + + default: /* unknown or supported type - oh dear */ + ret = EOF; + errno = EBADF; + break; + } + + free(file->buffer);/* free any allocated buffer space */ + free(file); + + return ret; +} + +char *url_fgets(char *ptr, size_t size, URL_FILE *file) +{ + size_t want = size - 1;/* always need to leave room for zero termination */ + size_t loop; + + switch(file->type) { + case CFTYPE_FILE: + ptr = fgets(ptr, (int)size, file->handle.file); + break; + + case CFTYPE_CURL: + fill_buffer(file, want); + + /* check if there's data in the buffer - if not fill either errored or + * EOF */ + if(!file->buffer_pos) + return NULL; + + /* ensure only available data is considered */ + if(file->buffer_pos < want) + want = file->buffer_pos; + + /*buffer contains data */ + /* look for newline or eof */ + for(loop = 0; loop < want; loop++) { + if(file->buffer[loop] == '\n') { + want = loop + 1;/* include newline */ + break; + } + } + + /* xfer data to caller */ + memcpy(ptr, file->buffer, want); + ptr[want] = 0;/* always null terminate */ + + use_buffer(file, want); + + break; + + default: /* unknown or supported type - oh dear */ + ptr = NULL; + errno = EBADF; + break; + } + + return ptr;/*success */ +} + diff --git a/src/interfaces/libpq-oauth/http-service.h b/src/interfaces/libpq-oauth/http-service.h new file mode 100644 index 00000000000..c96ecbad755 --- /dev/null +++ b/src/interfaces/libpq-oauth/http-service.h @@ -0,0 +1,52 @@ +/*------------------------------------------------------------------------- + * + * http-service. + * + * Definitions for HTTP service file + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/interfaces/libpq-oauth/oauth-curl.h + * + *------------------------------------------------------------------------- + */ + +#ifndef HTTP_SERVICE_H +#define HTTP_SERVICE_H + +#include "libpq-fe.h" + +enum fcurl_type_e { + CFTYPE_NONE = 0, + CFTYPE_FILE = 1, + CFTYPE_CURL = 2 +}; +struct fcurl_data +{ + enum fcurl_type_e type; /* type of handle */ + union { + CURL *curl; + FILE *file; + } handle; /* handle */ + + char *buffer; /* buffer to store cached data*/ + size_t buffer_len; /* currently allocated buffers length */ + size_t buffer_pos; /* end of data in buffer*/ + int still_running; /* Is background url fetch still in progress */ +}; + +typedef struct fcurl_data URL_FILE; + +/* Exported flow callback. */ + +extern PGDLLEXPORT URL_FILE +*url_fopen(const char *url, const char *operation); + +extern PGDLLEXPORT char +*url_fgets(char *ptr, size_t size, URL_FILE *file); + +extern PGDLLEXPORT int +url_fclose(URL_FILE *file); + +#endif /* HTTP_SERVICE_H */ diff --git a/src/interfaces/libpq-oauth/meson.build b/src/interfaces/libpq-oauth/meson.build index ea3a900f4f1..377955ed705 100644 --- a/src/interfaces/libpq-oauth/meson.build +++ b/src/interfaces/libpq-oauth/meson.build @@ -11,6 +11,7 @@ libpq_oauth_sources = files( # The shared library needs additional glue symbols. libpq_oauth_so_sources = files( 'oauth-utils.c', + 'http-service.c', ) libpq_oauth_so_c_args = [ '-DUSE_DYNAMIC_OAUTH', diff --git a/src/interfaces/libpq/fe-connect.c b/src/interfaces/libpq/fe-connect.c index db9b4c8edbf..d0c57b7351d 100644 --- a/src/interfaces/libpq/fe-connect.c +++ b/src/interfaces/libpq/fe-connect.c @@ -15,6 +15,7 @@ #include "postgres_fe.h" +#include #include #include #include @@ -142,6 +143,31 @@ static int ldapServiceLookup(const char *purl, PQconninfoOption *options, #define DefaultGSSMode "disable" #endif +#ifdef USE_LIBCURL +static int httpServiceLookup(const char *purl, PQconninfoOption *options, + PQExpBuffer errorMessage); +enum fcurl_type_e { + CFTYPE_NONE = 0, + CFTYPE_FILE = 1, + CFTYPE_CURL = 2 +}; +struct fcurl_data +{ + enum fcurl_type_e type; /* type of handle */ + union { + void *curl; + FILE *file; + } handle; /* handle */ + + char *buffer; /* buffer to store cached data*/ + size_t buffer_len; /* currently allocated buffers length */ + size_t buffer_pos; /* end of data in buffer*/ + int still_running; /* Is background url fetch still in progress */ +}; + +typedef struct fcurl_data URL_FILE; +#endif + /* ---------- * Definition of the conninfo parameters and their fallback resources. * @@ -5973,6 +5999,229 @@ ldapServiceLookup(const char *purl, PQconninfoOption *options, #endif /* USE_LDAP */ +#ifdef USE_LIBCURL +#define HTTP_URL "http://" + + +/* + * httpServiceLookup + * + * Search the HTTP URL passed as first argument, treat the result as a + * string of connection options that are parsed and added to the array of + * options passed as second argument. + * + * Returns + * 0 if the lookup was successful, + * 1 if the connection to the LDAP server could be established but + * the search was unsuccessful, + * 2 if a connection could not be established, and + * 3 if a fatal error occurred. + * 4 if libpq-oauth module does not exist + * + * An error message is appended to *errorMessage for return codes 1 and 3. + */ +static int +httpServiceLookup( +const char *purl, PQconninfoOption *options, + PQExpBuffer errorMessage + ) +{ + int result = 0, + linenr = 0, + i; + URL_FILE *f; + + char *line, + *url; + char buf[1024]; + + void *libcurl_module; + URL_FILE * (*url_fopen) (const char *url, const char *operation); + char * (*url_fgets) (char *ptr, size_t size, URL_FILE *file); + int (*url_fclose) (URL_FILE *file); + + const char *const module_name = +#if defined(__darwin__) + LIBDIR "/libpq-oauth" DLSUFFIX; +#else + "libpq-oauth" DLSUFFIX; +#endif + + if ((url = strdup(purl)) == NULL) + { + libpq_append_error(errorMessage, "out of memory"); + return 3; + } + + if (pg_strncasecmp(url, HTTP_URL, strlen(HTTP_URL)) != 0) + { + libpq_append_error(errorMessage, + "invalid HTTP URL \"%s\": scheme must be http://", purl); + free(url); + return 3; + } + + libcurl_module = dlopen(module_name, RTLD_NOW | RTLD_LOCAL); + if (!libcurl_module) + { + /* + * For end users, this probably isn't an error condition, it just + * means the flow isn't installed. Developers and package maintainers + * may want to debug this via the PGOAUTHDEBUG envvar, though. + * + * Note that POSIX dlerror() isn't guaranteed to be threadsafe. + */ + free(url); + if (oauth_unsafe_debugging_enabled()) + fprintf(stderr, "failed dlopen for libpq-oauth: %s\n", dlerror()); + return 4; + } + + if ((url_fopen = dlsym(libcurl_module, "url_fopen")) == NULL + || (url_fgets = dlsym(libcurl_module, "url_fgets")) == NULL + || (url_fclose = dlsym(libcurl_module, "url_fclose")) == NULL) + { + if (oauth_unsafe_debugging_enabled()) + fprintf(stderr, "failed dlsym for libpq-oauth: %s\n", dlerror()); + + dlclose(libcurl_module); + libcurl_module = NULL; + + libpq_append_error(errorMessage, + "could not find entry point for libpq-oauth"); + return 3; + } + + f = url_fopen(url, "r"); + if (f == NULL) + { + free(url); + return 3; + } + + free(url); + + /* assume connection could not be established until we read */ + result = 2; + + while ((line = url_fgets(buf, sizeof(buf), f)) != NULL) + { + int len; + char *key, + *val; + bool found_keyword; + + linenr++; + + if (strlen(line) >= sizeof(buf) - 1) + { + libpq_append_error(errorMessage, + "line %d too long in service file \"%s\"", + linenr, + purl); + result = 3; + goto exit; + } + + /* ignore whitespace at end of line, especially the newline */ + len = strlen(line); + while (len > 0 && isspace((unsigned char) line[len - 1])) + line[--len] = '\0'; + + /* ignore leading whitespace too */ + while (*line && isspace((unsigned char) line[0])) + line++; + + /* ignore comments and empty lines */ + if (line[0] == '\0' || line[0] == '#') + continue; + + /* + * we do not support ldap lookups within http lookups but we do raise erorr + * even in non ldap builds + */ + if (strncmp(line, "ldap", 4) == 0) + { + libpq_append_error(errorMessage, + "ldap:// lines are not allowed in http service lookups" + ); + result = 3; + goto exit; + } + + /* we do not support recursive http lookups */ + if (strncmp(line, "http", 4) == 0) + { + libpq_append_error(errorMessage, + "http:// lines are not allowed in http service lookups" + ); + result = 3; + goto exit; + } + + key = line; + val = strchr(line, '='); + if (val == NULL) + { + libpq_append_error(errorMessage, + "syntax error in service file \"%s\", line %d", + purl, + linenr); + result = 3; + goto exit; + } + *val++ = '\0'; + + if (strcmp(key, "service") == 0) + { + libpq_append_error(errorMessage, + "nested service specifications not supported in service file \"%s\", line %d", + purl, + linenr); + result = 3; + goto exit; + } + + /* + * Set the parameter --- but don't override any previous + * explicit setting. + */ + found_keyword = false; + for (i = 0; options[i].keyword; i++) + { + if (strcmp(options[i].keyword, key) == 0) + { + if (options[i].val == NULL) + options[i].val = strdup(val); + if (!options[i].val) + { + libpq_append_error(errorMessage, "out of memory"); + result = 3; + goto exit; + } + found_keyword = true; + break; + } + } + + if (!found_keyword) + { + libpq_append_error(errorMessage, + "syntax error in service file \"%s\", line %d", + purl, + linenr); + result = 3; + goto exit; + } + } + +exit: + url_fclose(f); + + return result; +} +#endif /* USE_LIBCURL */ + /* * parseServiceInfo: if a service name has been given, look it up and absorb * connection options from it into *options. @@ -6153,6 +6402,26 @@ parseServiceFile(const char *serviceFile, } #endif +#ifdef USE_LIBCURL + if (strncmp(line, "http", 4) == 0) + { + int rc = httpServiceLookup(line, options, errorMessage); + + /* if rc = 2 or 4, go on reading for fallback */ + switch (rc) + { + case 0: + goto exit; + case 1: + case 3: + result = 3; + goto exit; + case 2: + case 4: + continue; + } + } +#endif key = line; val = strchr(line, '='); if (val == NULL) diff --git a/src/test/modules/Makefile b/src/test/modules/Makefile index 28ce3b35eda..0b29fc3a557 100644 --- a/src/test/modules/Makefile +++ b/src/test/modules/Makefile @@ -13,6 +13,7 @@ SUBDIRS = \ index \ libpq_pipeline \ oauth_validator \ + http_pg_service \ plsample \ spgist_name_ops \ test_aio \ diff --git a/src/test/modules/http_pg_service/Makefile b/src/test/modules/http_pg_service/Makefile new file mode 100644 index 00000000000..f8ebc424dd8 --- /dev/null +++ b/src/test/modules/http_pg_service/Makefile @@ -0,0 +1,37 @@ +#------------------------------------------------------------------------- +# +# Makefile for src/test/modules/oauth_validator +# +# Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group +# Portions Copyright (c) 1994, Regents of the University of California +# +# src/test/modules/oauth_validator/Makefile +# +#------------------------------------------------------------------------- + +PGFILEDESC = "http_pg_service - test http_service module" + +PGAPPICON = win32 + +PG_CPPFLAGS = -I$(libpq_srcdir) +PG_LIBS_INTERNAL += $(libpq_pgport) + +NO_INSTALLCHECK = 1 + +TAP_TESTS = 1 + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = src/test/modules/http_pg_service +top_builddir = ../../../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk + +export PYTHON +export with_libcurl +export with_python + +endif diff --git a/src/test/modules/http_pg_service/meson.build b/src/test/modules/http_pg_service/meson.build new file mode 100644 index 00000000000..c8ace4aaa10 --- /dev/null +++ b/src/test/modules/http_pg_service/meson.build @@ -0,0 +1,17 @@ +# Copyright (c) 2025, PostgreSQL Global Development Group + +tests += { + 'name': 'http_pg_service', + 'sd': meson.current_source_dir(), + 'bd': meson.current_build_dir(), + 'tap': { + 'tests': [ + 't/001_http_service_file.pl', + ], + 'env': { + 'PYTHON': python.path(), + 'with_libcurl': libcurl.found() ? 'yes' : 'no', + 'with_python': 'yes', + }, + }, +} diff --git a/src/test/modules/http_pg_service/t/001_http_service_file.pl b/src/test/modules/http_pg_service/t/001_http_service_file.pl new file mode 100644 index 00000000000..264f1a8a882 --- /dev/null +++ b/src/test/modules/http_pg_service/t/001_http_service_file.pl @@ -0,0 +1,200 @@ +use strict; +use warnings FATAL => 'all'; + +use FindBin; +use lib $FindBin::RealBin; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +use PgHttpService::Server; + +if ($ENV{with_libcurl} ne 'yes') +{ + plan skip_all => 'HTTP service file not supported by this build'; +} + +if ($ENV{with_python} ne 'yes') +{ + plan skip_all => 'HTTP service tests require --with-python to run'; +} + +my $td = PostgreSQL::Test::Utils::tempdir; + +my $node_dummy = PostgreSQL::Test::Cluster->new('node_dummy'); +$node_dummy->init; + +my $node = PostgreSQL::Test::Cluster->new('node'); +$node->init; +$node->start; + +# Windows vs non-Windows: CRLF vs LF for the file's newline, relying on +# the fact that libpq uses fgets() when reading the lines of a service file. +my $newline = "\n"; + +# Create the set of service files used in the tests. +# File that includes a valid service name, that uses a decomposed connection +# string for its contents, split on spaces. +my $remote_srvfile_valid = "$td/remote_pg_service_valid.conf"; +append_to_file($remote_srvfile_valid, qq{ +port=} . $node->port() . qq{ +host=} . $node->host() . qq{ +dbname=postgres +}); + +my $missing_equals_invalid = "$td/missing_equals_invalid.conf"; +append_to_file($missing_equals_invalid, qq{ +port=} . $node->port() . qq{ +host=} . $node->host() . qq{ +dbname= +}); + +my $remote_pg_service_nonexistant_connection_option = "$td/remote_pg_service_nonexistant_connection_option.conf"; +append_to_file($remote_pg_service_nonexistant_connection_option, qq{ +port=} . $node->port() . qq{ +nonexistant_param=something +dbname=postgres +}); + +my $remote_pg_service_ldap_connection_option = "$td/remote_pg_service_ldap_connection_option.conf"; +append_to_file($remote_pg_service_ldap_connection_option, qq{ +ldap:// +}); + +my $remote_pg_service_http_connection_option = "$td/remote_pg_service_http_connection_option.conf"; +append_to_file($remote_pg_service_http_connection_option, qq{ +http:// +}); + +my $remote_pg_service_nested_connection_option = "$td/remote_pg_service_nested_connection_option.conf"; +append_to_file($remote_pg_service_nested_connection_option, qq{ +service=nonexistent +}); + +# File defined with no contents, used as default value for PGSERVICEFILE, +# so as no lookup is attempted in the user's home directory. +my $srvfile_empty = "$td/pg_service_empty.conf"; +append_to_file($srvfile_empty, ''); + +my $server = PgHttpService::Server->new(); +$server->run($td); + +my $port = $server->port; +my $issuer = "http://127.0.0.1:$port"; + +my $local_srvfile_valid = "$td/local_pg_service_valid.conf"; +append_to_file($local_srvfile_valid, qq{ +[http_service] +} . $issuer . qq{/remote_pg_service_valid.conf + +[http_service_incorrect_scheme_with_fallback] +http:/127.0.0.1:} . $port . qq{ +} . $issuer . qq{/remote_pg_service_valid.conf + +[http_service_incorrect_scheme] +http:/127.0.0.1:} . $port . qq{/remote_pg_service_valid.conf + +[http_service_non_listening_port] +http://127.0.0.1:1234/remote_pg_service_valid.conf + +[http_service_non_listening_port_with_fallback] +http://127.0.0.1:1234 +} . $issuer . qq{/remote_pg_service_valid.conf + +[remote_pg_service_nonexistant_connection_option] +} . $issuer . qq{/remote_pg_service_nonexistant_connection_option.conf + +[nested_http_service] +} . $issuer . qq{/remote_pg_service_http_connection_option.conf + +[ldap_service] +} . $issuer . qq{/remote_pg_service_ldap_connection_option.conf + +[remote_nested] +} . $issuer . qq{/remote_pg_service_nested_connection_option.conf + +[missing_equals_invalid] +} . $issuer . qq{/missing_equals_invalid.conf +}); + +# Set the fallback directory lookup of the service file to the temporary +# directory of this test. PGSYSCONFDIR is used if the service file +# defined in PGSERVICEFILE cannot be found, or when a service file is +# found but not the service name. +local $ENV{PGSYSCONFDIR} = $td; + +# Force PGSERVICEFILE to a default location, so as this test never +# tries to look at a home directory. This value needs to remain +# at the top of this script before running any tests, and should never +# be changed. + +{ + local $ENV{PGSERVICEFILE} = $local_srvfile_valid; + + $node_dummy->connect_ok( + 'service=http_service', + 'connection with correct "service" string and PGSERVICEFILE', + sql => "SELECT 'connect1_1'", + expected_stdout => qr/connect1_1/); + + $node_dummy->connect_ok( + 'postgres://?service=http_service', + 'connection with correct "service" URI and PGSERVICEFILE', + sql => "SELECT 'connect1_2'", + expected_stdout => qr/connect1_2/); + + $node_dummy->connect_fails( + 'service=http_service_incorrect_scheme', + 'service with invalid http scheme correctly raises error', + expected_stderr=> qr/psql: error: invalid HTTP URL \".*\": scheme must be http:\/\//); + + $node_dummy->connect_fails( + 'service=http_service_non_listening_port', + 'query on non listening port continues to use connection parameters from node_dummy', + expected_stderr=> + qr/psql: error: connection to server on socket \"\/tmp\/.*\" failed: No such file or directory/); + + $node_dummy->connect_ok( + 'service=http_service_non_listening_port_with_fallback', + 'check that non listening URL allows fallback to listening URL', + sql => "SELECT 'connect1_3'", + expected_stdout => qr/connect1_3/); + + $node_dummy->connect_fails( + 'service=http_service_incorrect_scheme_with_fallback', + 'check that incorrect http address does not allow fallback', + expected_stderr => qr/psql: error: invalid HTTP URL \".*\": scheme must be http:\/\//); + + $node_dummy->connect_fails( + 'service=remote_pg_service_nonexistant_connection_option', + '', + expected_stderr => qr/psql: error: syntax error in service file \"http:\/\/127\.0\.0\.1:.*\/remote_pg_service_nonexistant_connection_option.conf\"/); + + $node_dummy->connect_fails( + 'service=ldap_service', + 'check that ldap:// lines are not allowed in http service file', + expected_stderr => qr/psql: error: ldap:\/\/ lines are not allowed in http service lookups/); + + $node_dummy->connect_fails( + 'service=nested_http_service', + 'check that http:// lines are not allowed in http service file', + expected_stderr => qr/psql: error: http:\/\/ lines are not allowed in http service lookups/); + + $node_dummy->connect_fails( + 'service=remote_nested', + '', + expected_stdout => qr/psql: error: nested service specifications not supported in service file \"http:\/\/127\.0\.0\.1:.*\/remote_pg_service_nested_connection_option.conf\"/); + + $node_dummy->connect_fails( + 'service=missing_equals_invalid', + '', + expected_stdout => qr/psql: error: syntax error in service file \"http:\/\/127\.0\.0\.1:.*\/missing_equals_invalid.conf\"/); + +} + +$server->stop; + +$node->teardown_node; + +done_testing(); diff --git a/src/test/modules/http_pg_service/t/PgHttpService/Server.pm b/src/test/modules/http_pg_service/t/PgHttpService/Server.pm new file mode 100644 index 00000000000..e3d67af3b33 --- /dev/null +++ b/src/test/modules/http_pg_service/t/PgHttpService/Server.pm @@ -0,0 +1,141 @@ +# Copyright (c) 2025, PostgreSQL Global Development Group + +=pod + +=head1 NAME + +PgHttpService::Server - runs a mock pg_service HTTP server for testing + +=head1 SYNOPSIS + + use PgHttpService::Server; + + my $server = PgHttpService::Server->new(); + $server->run; + + my $port = $server->port; + my $issuer = "http://127.0.0.1:$port"; + + # test against $issuer... + + $server->stop; + +=head1 DESCRIPTION + +This is glue API between the Perl tests and the Python pg_service server +daemon implemented in t/http_service_server.py. (Python has a fairly usable HTTP server +in its standard library, so the implementation was ported from Perl.) + +This pg_service server does not use TLS (it implements a nonstandard, unsafe +issuer at "http://127.0.0.1:"), so libpq in particular will need to set +PGHTTPSERVICEBUG=UNSAFE to be able to talk to it. + +=cut + +package PgHttpService::Server; + +use warnings; +use strict; +use Scalar::Util; +use Test::More; + +=pod + +=head1 METHODS + +=over + +=item PgHttpService::Server::Server->new() + +Create a new HTTP PG Service Server object. + +=cut + +sub new +{ + my $class = shift; + + my $self = {}; + bless($self, $class); + + return $self; +} + +=pod + +=item $server->port() + +Returns the port in use by the server. + +=cut + +sub port +{ + my $self = shift; + + return $self->{'port'}; +} + +=pod + +=item $server->run() + +Runs the http pg service server daemon in t/http_service_server.py. + +=cut + +sub run +{ + my $self = shift; + my $service_file_path = shift; + my $port; + + print $ENV{PYTHON}; + my $pid = open(my $read_fh, "-|", "python", "t/http_service_server.py", $service_file_path) + or die "failed to start http pg_service server: $!"; + + # Get the port number from the daemon. It closes stdout afterwards; that way + # we can slurp in the entire contents here rather than worrying about the + # number of bytes to read. + $port = do { local $/ = undef; <$read_fh> } + // die "failed to read port number: $!"; + chomp $port; + die "server did not advertise a valid port" + unless Scalar::Util::looks_like_number($port); + + $self->{'pid'} = $pid; + $self->{'port'} = $port; + $self->{'child'} = $read_fh; + + note("HTTP pg_service (PID $pid) is listening on port $port\n"); +} + +=pod + +=item $server->stop() + +Sends SIGTERM to the http pg_service server and waits for it to exit. + +=cut + +sub stop +{ + my $self = shift; + + note("Sending SIGTERM to http pg_service PID: $self->{'pid'}\n"); + + kill(15, $self->{'pid'}); + $self->{'pid'} = undef; + + # Closing the popen() handle waits for the process to exit. + close($self->{'child'}); + $self->{'child'} = undef; +} + +=pod + +=back + +=cut + +1; diff --git a/src/test/modules/http_pg_service/t/http_service_server.py b/src/test/modules/http_pg_service/t/http_service_server.py new file mode 100644 index 00000000000..7fd44f24da0 --- /dev/null +++ b/src/test/modules/http_pg_service/t/http_service_server.py @@ -0,0 +1,73 @@ +#! /usr/bin/env python3 +# +# A mock http pg_service server, designed to be invoked from +# PgHttpService/Server.pm. This listens on an ephemeral port number (printed to stdout +# so that the Perl tests can contact it) and runs as a daemon until it is +# signaled. +# + +import http.server +import os +import sys +import textwrap + + +class ServiceFileHandler(http.server.BaseHTTPRequestHandler): + """ + Core implementation of the service file server. The API is + inheritance-based, with an entry point at do_GET(). See the + documentation for BaseHTTPRequestHandler. + """ + + JsonObject = dict[str, object] # TypeAlias is not available until 3.10 + + + def do_GET(self): + self._response_code = 200 + + self._send_service_file() + + def _send_service_file(self) -> None: + """ + Sends the provided JSON dict as an application/json response. + self._response_code can be modified to send JSON error responses. + """ + + service_file_path = sys.argv[1] + "/" + self.path + with open(service_file_path, "r") as file: + service_file_content = file.read() + + resp = service_file_content.encode() + self.log_message("sending string response: %s", resp) + + self.send_response(self._response_code) + self.send_header("Content-Type", "text/plain") + self.send_header("Content-Length", str(len(resp))) + self.end_headers() + + self.wfile.write(resp) + + +def main(): + """ + Starts the PgHttpService server on localhost. The ephemeral port in use will + be printed to stdout. + """ + + s = http.server.HTTPServer(("127.0.0.1", 0), ServiceFileHandler) + + # Give the parent the port number to contact (this is also the signal that + # we're ready to receive requests). + port = s.socket.getsockname()[1] + print(port) + + # stdout is closed to allow the parent to just "read to the end". + stdout = sys.stdout.fileno() + sys.stdout.close() + os.close(stdout) + + s.serve_forever() # we expect our parent to send a termination signal + + +if __name__ == "__main__": + main() -- 2.51.2