From e161270b1db1b5206e2d5b560de38eebf2f99560 Mon Sep 17 00:00:00 2001 From: Jakub Wartak Date: Fri, 24 Apr 2026 13:09:01 +0200 Subject: [PATCH v1 2/2] Add BLACKHOLE destination format in pg_dump (-Fb) Enable BLACKHOLE in pg_dump along with parlallel mode. This allows for a parallelized fast logical check of the whole database if all of the relations are readable. In addition it does not require the free disk space to write output files, contrary to the directory format. Author: Jakub Wartak Reviewed-by: Discussion: --- src/bin/pg_dump/Makefile | 1 + src/bin/pg_dump/meson.build | 1 + src/bin/pg_dump/pg_backup.h | 1 + src/bin/pg_dump/pg_backup_archiver.c | 4 + src/bin/pg_dump/pg_backup_archiver.h | 1 + src/bin/pg_dump/pg_backup_blackhole.c | 185 ++++++++++++++++++++++++++ src/bin/pg_dump/pg_dump.c | 49 +++++-- 7 files changed, 234 insertions(+), 8 deletions(-) create mode 100644 src/bin/pg_dump/pg_backup_blackhole.c diff --git a/src/bin/pg_dump/Makefile b/src/bin/pg_dump/Makefile index 79073b0a0ea..d38a2dd7eba 100644 --- a/src/bin/pg_dump/Makefile +++ b/src/bin/pg_dump/Makefile @@ -36,6 +36,7 @@ OBJS = \ filter.o \ parallel.o \ pg_backup_archiver.o \ + pg_backup_blackhole.o \ pg_backup_custom.o \ pg_backup_db.o \ pg_backup_directory.o \ diff --git a/src/bin/pg_dump/meson.build b/src/bin/pg_dump/meson.build index 7c9a475963b..82786b4b8bc 100644 --- a/src/bin/pg_dump/meson.build +++ b/src/bin/pg_dump/meson.build @@ -11,6 +11,7 @@ pg_dump_common_sources = files( 'filter.c', 'parallel.c', 'pg_backup_archiver.c', + 'pg_backup_blackhole.c', 'pg_backup_custom.c', 'pg_backup_db.c', 'pg_backup_directory.c', diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index 28e7ff6fa16..0c5c5bd34a2 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -43,6 +43,7 @@ typedef enum _archiveFormat archTar = 3, archNull = 4, archDirectory = 5, + archBlackhole = 6, } ArchiveFormat; typedef enum _archiveMode diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index 2fd773ad84f..2dd7ea88913 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -2505,6 +2505,10 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, InitArchiveFmt_Tar(AH); break; + case archBlackhole: + InitArchiveFmt_Blackhole(AH); + break; + default: pg_fatal("unrecognized file format \"%d\"", AH->format); } diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h index 1218bf6a6a1..cbad5d7cb69 100644 --- a/src/bin/pg_dump/pg_backup_archiver.h +++ b/src/bin/pg_dump/pg_backup_archiver.h @@ -464,6 +464,7 @@ extern void InitArchiveFmt_Custom(ArchiveHandle *AH); extern void InitArchiveFmt_Null(ArchiveHandle *AH); extern void InitArchiveFmt_Directory(ArchiveHandle *AH); extern void InitArchiveFmt_Tar(ArchiveHandle *AH); +extern void InitArchiveFmt_Blackhole(ArchiveHandle *AH); extern void ReconnectToServer(ArchiveHandle *AH, const char *dbname); extern void IssueCommandPerBlob(ArchiveHandle *AH, TocEntry *te, diff --git a/src/bin/pg_dump/pg_backup_blackhole.c b/src/bin/pg_dump/pg_backup_blackhole.c new file mode 100644 index 00000000000..5b596dbb250 --- /dev/null +++ b/src/bin/pg_dump/pg_backup_blackhole.c @@ -0,0 +1,185 @@ +/*------------------------------------------------------------------------- + * + * pg_backup_blackhole.c + * + * Implementation of an archive that is never saved and never outputs anything. + * It is used by pg_dump to execute COPY TO BLACKHOLE commands. + * + * IDENTIFICATION + * src/bin/pg_dump/pg_backup_blackhole.c + * + *------------------------------------------------------------------------- + */ +#include "postgres_fe.h" + +#include "parallel.h" +#include "pg_backup_archiver.h" +#include "pg_backup_utils.h" + +static int _WorkerJobDumpDirectory(ArchiveHandle *AH, TocEntry *te); +static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te); +static void _StartData(ArchiveHandle *AH, TocEntry *te); +static void _Clone(ArchiveHandle *AH); +static void _DeClone(ArchiveHandle *AH); +static void _WriteData(ArchiveHandle *AH, const void *data, size_t dLen); +static void _EndData(ArchiveHandle *AH, TocEntry *te); +static int _WriteByte(ArchiveHandle *AH, const int i); +static void _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len); +static void _CloseArchive(ArchiveHandle *AH); +static void _PrintTocData(ArchiveHandle *AH, TocEntry *te); +static void _StartLOs(ArchiveHandle *AH, TocEntry *te); +static void _StartLO(ArchiveHandle *AH, TocEntry *te, Oid oid); +static void _EndLO(ArchiveHandle *AH, TocEntry *te, Oid oid); +static void _EndLOs(ArchiveHandle *AH, TocEntry *te); + +/* + * Initializer + */ +void +InitArchiveFmt_Blackhole(ArchiveHandle *AH) +{ + /* Assuming static functions, this can be copied for each format. */ + AH->ArchiveEntryPtr = _ArchiveEntry; + AH->StartDataPtr = _StartData; + + AH->WriteDataPtr = _WriteData; + AH->EndDataPtr = _EndData; + AH->WriteBytePtr = _WriteByte; + AH->WriteBufPtr = _WriteBuf; + AH->ClosePtr = _CloseArchive; + AH->ReopenPtr = NULL; + AH->PrintTocDataPtr = _PrintTocData; + + AH->StartLOsPtr = _StartLOs; + AH->StartLOPtr = _StartLO; + AH->EndLOPtr = _EndLO; + AH->EndLOsPtr = _EndLOs; + + AH->ClonePtr = _Clone; + AH->DeClonePtr = _DeClone; + + /* no parallel dump in the custom archive, only parallel restore */ + AH->WorkerJobDumpPtr = _WorkerJobDumpDirectory; + + if (AH->mode == archModeRead) + pg_fatal("this format cannot be read"); +} + +static int +_WorkerJobDumpDirectory(ArchiveHandle *AH, TocEntry *te) +{ + WriteDataChunksForTocEntry(AH, te); + + /* Return nothing */ + return 0; +} + +/* + * Those must be non-NULL, because CloneArchive() / DeCloneArchive() invokes + * them. + */ +static void +_Clone(ArchiveHandle *AH) +{ + /* Do nothing */ +} + +static void +_DeClone(ArchiveHandle *AH) +{ + /* Do nothing */ +} + +static void +_ArchiveEntry(ArchiveHandle *AH, TocEntry *te) +{ + /* Do nothing */ +} + +static void +_StartData(ArchiveHandle *AH, TocEntry *te) +{ + /* Do nothing */ +} + +static void +_WriteData(ArchiveHandle *AH, const void *data, size_t dLen) +{ + /* Do nothing */ +} + +static void +_EndData(ArchiveHandle *AH, TocEntry *te) +{ + /* Do nothing */ +} + +static void +_StartLOs(ArchiveHandle *AH, TocEntry *te) +{ + /* Do nothing */ +} + +static void +_StartLO(ArchiveHandle *AH, TocEntry *te, Oid oid) +{ + /* Do nothing */ +} + +static void +_EndLO(ArchiveHandle *AH, TocEntry *te, Oid oid) +{ + /* Do nothing */ +} + +static void +_EndLOs(ArchiveHandle *AH, TocEntry *te) +{ + /* Do nothing */ +} + +static void +_PrintTocData(ArchiveHandle *AH, TocEntry *te) +{ + if (te->dataDumper) + { + AH->currToc = te; + te->dataDumper((Archive *) AH, te->dataDumperArg); + AH->currToc = NULL; + } +} + +static int +_WriteByte(ArchiveHandle *AH, const int i) +{ + return 0; +} + +static void +_WriteBuf(ArchiveHandle *AH, const void *buf, size_t len) +{ + /* Do nothing */ +} + +/* + * Close the archive. + * + * When writing the archive, this is the routine that actually starts + * the process of saving it to files. + */ +static void +_CloseArchive(ArchiveHandle *AH) +{ + ParallelState *pstate; + + if (AH->mode != archModeWrite) + return; + + /* + * WriteDataChunks() calls TocEntry's dataDumper (dumpTableData_copy) that + * issues COPY table TO BLACKHOLE. + */ + pstate = ParallelBackupStart(AH); + WriteDataChunks(AH, pstate); + ParallelBackupEnd(AH, pstate); +} diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index d56dcc701ce..3950dc56227 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -958,8 +958,11 @@ main(int argc, char **argv) if (!plainText) dopt.outputCreateDB = 1; - /* Parallel backup only in the directory archive format so far */ - if (archiveFormat != archDirectory && numWorkers > 1) + /* + * Parallel backup only in the BLACKHOLE or the directory archive format + * so far + */ + if (numWorkers > 1 && archiveFormat != archDirectory && archiveFormat != archBlackhole) pg_fatal("parallel backup only supported by the directory format"); /* Open the output file */ @@ -1296,8 +1299,8 @@ help(const char *progname) printf(_("\nGeneral options:\n")); printf(_(" -f, --file=FILENAME output file or directory name\n")); - printf(_(" -F, --format=c|d|t|p output file format (custom, directory, tar,\n" - " plain text (default))\n")); + printf(_(" -F, --format=c|d|t|p|b output file format (custom, directory, tar,\n" + " plain text (default), blackhole)\n")); printf(_(" -j, --jobs=NUM use this many parallel jobs to dump\n")); printf(_(" -v, --verbose verbose mode\n")); printf(_(" -V, --version output version information, then exit\n")); @@ -1634,6 +1637,8 @@ parseArchiveFormat(const char *format, ArchiveMode *mode) archiveFormat = archTar; else if (pg_strcasecmp(format, "tar") == 0) archiveFormat = archTar; + else if (pg_strcasecmp(format, "b") == 0 || pg_strcasecmp(format, "blackhole") == 0) + archiveFormat = archBlackhole; else pg_fatal("invalid output format \"%s\" specified", format); return archiveFormat; @@ -2367,6 +2372,7 @@ dumpTableData_copy(Archive *fout, const void *dcontext) const TableInfo *tbinfo = tdinfo->tdtable; const char *classname = tbinfo->dobj.name; PQExpBuffer q = createPQExpBuffer(); + ArchiveHandle *AH = (ArchiveHandle *) fout; /* * Note: can't use getThreadLocalPQExpBuffer() here, we're calling fmtId @@ -2378,6 +2384,14 @@ dumpTableData_copy(Archive *fout, const void *dcontext) int ret; char *copybuf; const char *column_list; + char *copy_dest = "stdout"; + bool blackhole = false; + + if (AH->format == archBlackhole) + { + copy_dest = "BLACKHOLE"; + blackhole = true; + } pg_log_info("dumping contents of table \"%s.%s\"", tbinfo->dobj.namespace->dobj.name, classname); @@ -2414,16 +2428,35 @@ dumpTableData_copy(Archive *fout, const void *dcontext) else appendPQExpBufferStr(q, "* "); - appendPQExpBuffer(q, "FROM %s %s) TO stdout;", + appendPQExpBuffer(q, "FROM %s %s) TO %s;", fmtQualifiedDumpable(tbinfo), - tdinfo->filtercond ? tdinfo->filtercond : ""); + tdinfo->filtercond ? tdinfo->filtercond : "", + copy_dest); } else { - appendPQExpBuffer(q, "COPY %s %s TO stdout;", + appendPQExpBuffer(q, "COPY %s %s TO %s;", fmtQualifiedDumpable(tbinfo), - column_list); + column_list, + copy_dest); } + + /* + * COPY TO BLACKHOLE discards rows server-side and never sends a + * CopyOutResponse, so it completes with PGRES_COMMAND_OK and there is no + * client-side message to receive. + */ + if (blackhole) + { + res = ExecuteSqlQuery(fout, q->data, PGRES_COMMAND_OK); + PQclear(res); + destroyPQExpBuffer(clistBuf); + destroyPQExpBuffer(q); + if (tbinfo->relkind == RELKIND_FOREIGN_TABLE) + set_restrict_relation_kind(fout, "view, foreign-table"); + return 1; + } + res = ExecuteSqlQuery(fout, q->data, PGRES_COPY_OUT); PQclear(res); destroyPQExpBuffer(clistBuf); -- 2.43.0