From ab81384954c756134eb417fda656be0f23ebc498 Mon Sep 17 00:00:00 2001 From: Jakub Wartak Date: Thu, 30 Jul 2026 11:02:42 +0200 Subject: [PATCH v06082026 02/10] pg_basebackup: add new "client-blackhole" (benchmarking) backup target Accept a new "client-blackhole" target value for pg_basebackup's --target option. Unlike the server-side "server-blackhole" target, the whole backup is sent to pg_basebackup as usual, but instead of being written to disk it is discarded: regular file contents go to the /dev/null device and directories/symbolic links/backup manifest are not created. In tar format the archive itself is sent to the /dev/null device too. This is a testing and development feature. It allows measuring how fast a backup can be produced by the server and network without being limited by the speed of the local storage (but subject to pg_basebackup single-threaded limitations like TLS decryption and/or checksum validation). The client-side discard is implemented in the astreamer extractors, which is controlled by a new backup_target_clientblackhole flag (backup_target stays NULL). Author: Jakub Wartak --- doc/src/sgml/ref/pg_basebackup.sgml | 7 +- src/bin/pg_basebackup/pg_basebackup.c | 115 +++++++++++++++---- src/bin/pg_basebackup/t/010_pg_basebackup.pl | 18 +++ src/fe_utils/astreamer_file.c | 36 ++++-- src/include/fe_utils/astreamer.h | 3 +- 5 files changed, 150 insertions(+), 29 deletions(-) diff --git a/doc/src/sgml/ref/pg_basebackup.sgml b/doc/src/sgml/ref/pg_basebackup.sgml index 2ed643f005a..9ac679a1133 100644 --- a/doc/src/sgml/ref/pg_basebackup.sgml +++ b/doc/src/sgml/ref/pg_basebackup.sgml @@ -269,7 +269,12 @@ PostgreSQL documentation server requires superuser privileges or having privileges of the pg_write_server_files role. If the target is set to server-blackhole, the contents are discarded by the - server and not stored anywhere. This should only be used for testing + server and not stored anywhere. If the target is set to + client-blackhole, the backup is sent to + pg_basebackup as usual, but discarded instead + of being written to disk; this can be used to measure how fast a backup + can be produced without being limited by the speed of the local + storage. Both blackhole targets should only be used for testing purposes, as you will not end up with an actual backup. diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c index 8a599fc9869..41cb3f26855 100644 --- a/src/bin/pg_basebackup/pg_basebackup.c +++ b/src/bin/pg_basebackup/pg_basebackup.c @@ -155,6 +155,7 @@ static bool manifest = true; static bool manifest_force_encode = false; static char *manifest_checksums = NULL; static DataDirSyncMethod sync_method = DATA_DIR_SYNC_METHOD_FSYNC; +static bool backup_target_clientblackhole = false; static bool success = false; static bool made_new_pgdata = false; @@ -1151,7 +1152,8 @@ CreateBackupStreamer(char *archive_name, char *spclocation, directory = get_tablespace_mapping(spclocation); streamer = astreamer_extractor_new(directory, get_tablespace_mapping, - progress_update_filename); + progress_update_filename, + backup_target_clientblackhole); } else { @@ -1163,8 +1165,15 @@ CreateBackupStreamer(char *archive_name, char *spclocation, * Normally, we write it to the archive name provided by the caller, * but when the base directory is "-" that means we need to write to * standard output. + * + * When discarding writes, we send the archive to the null device. */ - if (strcmp(basedir, "-") == 0) + if (backup_target_clientblackhole) + { + snprintf(archive_filename, sizeof(archive_filename), "%s", DEVNULL); + archive_file = NULL; + } + else if (strcmp(basedir, "-") == 0) { snprintf(archive_filename, sizeof(archive_filename), "-"); archive_file = stdout; @@ -1176,25 +1185,32 @@ CreateBackupStreamer(char *archive_name, char *spclocation, archive_file = NULL; } + /* + * Setup streamer. In case of using client blackhole, it does not make sense + * to append compression suffixes. + */ if (compress->algorithm == PG_COMPRESSION_NONE) streamer = astreamer_plain_writer_new(archive_filename, archive_file); else if (compress->algorithm == PG_COMPRESSION_GZIP) { - strlcat(archive_filename, ".gz", sizeof(archive_filename)); + if (!backup_target_clientblackhole) + strlcat(archive_filename, ".gz", sizeof(archive_filename)); streamer = astreamer_gzip_writer_new(archive_filename, archive_file, compress); } else if (compress->algorithm == PG_COMPRESSION_LZ4) { - strlcat(archive_filename, ".lz4", sizeof(archive_filename)); + if (!backup_target_clientblackhole) + strlcat(archive_filename, ".lz4", sizeof(archive_filename)); streamer = astreamer_plain_writer_new(archive_filename, archive_file); streamer = astreamer_lz4_compressor_new(streamer, compress); } else if (compress->algorithm == PG_COMPRESSION_ZSTD) { - strlcat(archive_filename, ".zst", sizeof(archive_filename)); + if (!backup_target_clientblackhole) + strlcat(archive_filename, ".zst", sizeof(archive_filename)); streamer = astreamer_plain_writer_new(archive_filename, archive_file); streamer = astreamer_zstd_compressor_new(streamer, compress); @@ -1475,6 +1491,15 @@ ReceiveArchiveStreamChunk(size_t r, char *copybuf, void *callback_data) */ if (state->manifest_inject_streamer != NULL) state->manifest_buffer = createPQExpBuffer(); + else if (backup_target_clientblackhole) + { + /* Throw away the manifest too */ + snprintf(state->manifest_filename, + sizeof(state->manifest_filename), "%s", DEVNULL); + state->manifest_file = fopen(DEVNULL, "wb"); + if (state->manifest_file == NULL) + pg_fatal("could not open file \"%s\": %m", DEVNULL); + } else { snprintf(state->manifest_filename, @@ -1688,11 +1713,22 @@ ReceiveBackupManifest(PGconn *conn) { WriteManifestState state; - snprintf(state.filename, sizeof(state.filename), - "%s/backup_manifest.tmp", basedir); - state.file = fopen(state.filename, "wb"); - if (state.file == NULL) - pg_fatal("could not create file \"%s\": %m", state.filename); + if (backup_target_clientblackhole) + { + /* Throw away the manifest */ + snprintf(state.filename, sizeof(state.filename), "%s", DEVNULL); + state.file = fopen(DEVNULL, "wb"); + if (state.file == NULL) + pg_fatal("could not open file \"%s\": %m", DEVNULL); + } + else + { + snprintf(state.filename, sizeof(state.filename), + "%s/backup_manifest.tmp", basedir); + state.file = fopen(state.filename, "wb"); + if (state.file == NULL) + pg_fatal("could not create file \"%s\": %m", state.filename); + } ReceiveCopyData(conn, ReceiveBackupManifestChunk, &state); @@ -1975,6 +2011,9 @@ BaseBackup(char *compression_algorithm, char *compression_detail, compression_detail); } + if (verbose && backup_target_clientblackhole) + pg_log_info("the backup is being discarded and cannot be used for recovery"); + if (verbose) pg_log_info("initiating base backup, waiting for checkpoint to complete"); @@ -2057,7 +2096,8 @@ BaseBackup(char *compression_algorithm, char *compression_detail, * won't be storing anything into these directories and thus should * not create them. */ - if (backup_target == NULL && format == 'p' && !PQgetisnull(res, i, 1)) + if (backup_target == NULL && !backup_target_clientblackhole && format == 'p' && + !PQgetisnull(res, i, 1)) { char *path = PQgetvalue(res, i, 1); @@ -2284,11 +2324,8 @@ BaseBackup(char *compression_algorithm, char *compression_detail, * synced after being completed. In plain format, all the data of the * base directory is synced, taking into account all the tablespaces. * Errors are not considered fatal. - * - * If, however, there's a backup target, we're not writing anything - * locally, so in that case we skip this step. */ - if (do_sync && backup_target == NULL) + if (do_sync && backup_target == NULL && !backup_target_clientblackhole) { if (verbose) pg_log_info("syncing data to disk ..."); @@ -2310,7 +2347,7 @@ BaseBackup(char *compression_algorithm, char *compression_detail, * without a backup_manifest file, decreasing the chances that a directory * we leave behind will be mistaken for a valid backup. */ - if (!writing_to_stdout && manifest && backup_target == NULL) + if (!writing_to_stdout && manifest && backup_target == NULL && !backup_target_clientblackhole) { char tmp_filename[MAXPGPATH]; char filename[MAXPGPATH]; @@ -2487,7 +2524,15 @@ main(int argc, char **argv) temp_replication_slot = false; break; case 't': - backup_target = pg_strdup(optarg); + + /* + * Target: everything else than "client-blackhole" is passed + * through to the server as the backup target. + */ + if (strcmp(optarg, "client-blackhole") == 0) + backup_target_clientblackhole = true; + else + backup_target = pg_strdup(optarg); break; case 'T': tablespace_list_append(optarg); @@ -2587,6 +2632,17 @@ main(int argc, char **argv) backup_target = NULL; } + if (backup_target_clientblackhole) + { + if (basedir != NULL || backup_target != NULL) + { + pg_log_error("cannot specify both output directory and backup target"); + pg_log_error_hint("Try \"%s --help\" for more information.", progname); + exit(1); + } + basedir = pg_strdup(DEVNULL); + } + /* * Can't use --format with --target. Without --target, default format is * tar. @@ -2679,6 +2735,23 @@ main(int argc, char **argv) exit(1); } + /* + * The "client-blackhole" target throws the backup away as it is received. + * As we do not have directories prepared, we cannot stream or write recovery + * configuration. + */ + if (backup_target_clientblackhole) + { + if (includewal == STREAM_WAL) + { + pg_log_error("WAL cannot be streamed with the \"client-blackhole\" backup target"); + pg_log_error_hint("Use \"%s\" or \"%s\".", "-X none", "-X fetch"); + exit(1); + } + if (writerecoveryconf) + pg_fatal("recovery configuration cannot be written with the \"client-blackhole\" backup target"); + } + /* * Sanity checks for WAL method. */ @@ -2832,9 +2905,11 @@ main(int argc, char **argv) /* * If an output directory was specified, verify that it exists, or create * it. Note that for a tar backup, an output directory of "-" means we are - * writing to stdout, so do nothing in that case. + * writing to stdout, so do nothing in that case. When discarding writes, + * we don't create anything on disk at all. */ - if (basedir != NULL && (format == 'p' || strcmp(basedir, "-") != 0)) + if (basedir != NULL && !backup_target_clientblackhole && + (format == 'p' || strcmp(basedir, "-") != 0)) verify_dir_is_empty_or_create(basedir, &made_new_pgdata, &found_existing_pgdata); /* determine remote server's xlog segment size */ @@ -2842,7 +2917,7 @@ main(int argc, char **argv) exit(1); /* Create pg_wal symlink, if required */ - if (xlog_dir) + if (xlog_dir && !backup_target_clientblackhole) { char *linkloc; diff --git a/src/bin/pg_basebackup/t/010_pg_basebackup.pl b/src/bin/pg_basebackup/t/010_pg_basebackup.pl index 1b593ea73fe..03b05df85ad 100644 --- a/src/bin/pg_basebackup/t/010_pg_basebackup.pl +++ b/src/bin/pg_basebackup/t/010_pg_basebackup.pl @@ -324,6 +324,24 @@ $node->command_ok( ok(-f "$tempdir/tarbackup/base.tar", 'backup tar was created'); rmtree("$tempdir/tarbackup"); +# The "client-blackhole" target receives the whole backup but throws it away. +$node->command_ok( + [ + @pg_basebackup_defs, + '--target' => 'client-blackhole', + '--format' => 'plain', + '--wal-method' => 'none' + ], + 'client-blackhole target in plain format'); +$node->command_ok( + [ + @pg_basebackup_defs, + '--target' => 'client-blackhole', + '--format' => 'tar', + '--wal-method' => 'fetch' + ], + 'client-blackhole target in tar format'); + $node->command_fails_like( [ @pg_basebackup_defs, diff --git a/src/fe_utils/astreamer_file.c b/src/fe_utils/astreamer_file.c index fb36cecc22a..e4fa2596b08 100644 --- a/src/fe_utils/astreamer_file.c +++ b/src/fe_utils/astreamer_file.c @@ -37,6 +37,7 @@ typedef struct astreamer_extractor void (*report_output_file) (const char *); char filename[MAXPGPATH]; FILE *file; + bool discard_backup; } astreamer_extractor; static void astreamer_plain_writer_content(astreamer *streamer, @@ -60,7 +61,8 @@ static void astreamer_extractor_finalize(astreamer *streamer); static void astreamer_extractor_free(astreamer *streamer); static void extract_directory(const char *filename, mode_t mode); static void extract_link(const char *filename, const char *linktarget); -static FILE *create_file_for_extract(const char *filename, mode_t mode); +static FILE *create_file_for_extract(const char *filename, mode_t mode, + bool discard_backup); static const astreamer_ops astreamer_extractor_ops = { .content = astreamer_extractor_content, @@ -181,11 +183,15 @@ astreamer_plain_writer_free(astreamer *streamer) * 'report_output_file' is a function that will be called each time we open a * new output file. The pathname to that file is passed as an argument. If * NULL, the call is skipped. + * + * If 'discard_backup' is true, the extracted archive is thrown away rather + * than written to the filesystem. */ astreamer * astreamer_extractor_new(const char *basepath, const char *(*link_map) (const char *), - void (*report_output_file) (const char *)) + void (*report_output_file) (const char *), + bool discard_backup) { astreamer_extractor *streamer; @@ -195,6 +201,7 @@ astreamer_extractor_new(const char *basepath, streamer->basepath = pstrdup(basepath); streamer->link_map = link_map; streamer->report_output_file = report_output_file; + streamer->discard_backup = discard_backup; return &streamer->base; } @@ -231,13 +238,19 @@ astreamer_extractor_content(astreamer *streamer, astreamer_member *member, if (mystreamer->filename[fnamelen - 1] == '/') mystreamer->filename[fnamelen - 1] = '\0'; - /* Dispatch based on file type. */ + /* + * Dispatch based on file type. + */ if (member->is_regular) mystreamer->file = create_file_for_extract(mystreamer->filename, - member->mode); + member->mode, + mystreamer->discard_backup); else if (member->is_directory) - extract_directory(mystreamer->filename, member->mode); + { + if (!mystreamer->discard_backup) + extract_directory(mystreamer->filename, member->mode); + } else if (member->is_symlink) { const char *linktarget = member->linktarget; @@ -252,7 +265,8 @@ astreamer_extractor_content(astreamer *streamer, astreamer_member *member, member->linktarget); } - extract_link(mystreamer->filename, linktarget); + if (!mystreamer->discard_backup) + extract_link(mystreamer->filename, linktarget); } /* Report output file change. */ @@ -369,10 +383,18 @@ extract_link(const char *filename, const char *linktarget) * Return the resulting handle so we can write the content to the file. */ static FILE * -create_file_for_extract(const char *filename, mode_t mode) +create_file_for_extract(const char *filename, mode_t mode, bool discard_backup) { FILE *file; + if (discard_backup) + { + file = fopen(DEVNULL, "wb"); + if (file == NULL) + pg_fatal("could not open file \"%s\": %m", DEVNULL); + return file; + } + file = fopen(filename, "wb"); if (file == NULL) pg_fatal("could not create file \"%s\": %m", filename); diff --git a/src/include/fe_utils/astreamer.h b/src/include/fe_utils/astreamer.h index 8329e4efbc5..7206dc0c48b 100644 --- a/src/include/fe_utils/astreamer.h +++ b/src/include/fe_utils/astreamer.h @@ -215,7 +215,8 @@ extern astreamer *astreamer_gzip_writer_new(char *pathname, FILE *file, pg_compress_specification *compress); extern astreamer *astreamer_extractor_new(const char *basepath, const char *(*link_map) (const char *), - void (*report_output_file) (const char *)); + void (*report_output_file) (const char *), + bool discard_backup); extern astreamer *astreamer_gzip_decompressor_new(astreamer *next); extern astreamer *astreamer_lz4_compressor_new(astreamer *next, -- 2.43.0