From a4fa522d0259e8969cde32798a917321cced0415 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Thu, 1 Dec 2022 11:03:41 +0900 Subject: [PATCH v12] Teach pg_dump about compress_spec and use it throughout. Align pg_dump with the rest of the binaries which use common compression. It is teaching pg_dump.c about the common compression definitions and interfaces. Then it propagates those throughout the code. --- src/bin/pg_dump/compress_io.c | 107 ++++++++------------ src/bin/pg_dump/compress_io.h | 20 ++-- src/bin/pg_dump/pg_backup.h | 7 +- src/bin/pg_dump/pg_backup_archiver.c | 76 +++++++++----- src/bin/pg_dump/pg_backup_archiver.h | 10 +- src/bin/pg_dump/pg_backup_custom.c | 6 +- src/bin/pg_dump/pg_backup_directory.c | 13 ++- src/bin/pg_dump/pg_backup_tar.c | 11 +- src/bin/pg_dump/pg_dump.c | 67 +++++++----- src/bin/pg_dump/t/001_basic.pl | 34 +++++-- src/bin/pg_dump/t/002_pg_dump.pl | 3 +- src/test/modules/test_pg_dump/t/001_base.pl | 16 +++ doc/src/sgml/ref/pg_dump.sgml | 34 +++++-- src/tools/pgindent/typedefs.list | 1 - 14 files changed, 242 insertions(+), 163 deletions(-) diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c index 62f940ff7a..8f0d6d6210 100644 --- a/src/bin/pg_dump/compress_io.c +++ b/src/bin/pg_dump/compress_io.c @@ -64,7 +64,7 @@ /* typedef appears in compress_io.h */ struct CompressorState { - CompressionAlgorithm comprAlg; + pg_compress_specification compression_spec; WriteFunc writeF; #ifdef HAVE_LIBZ @@ -74,9 +74,6 @@ struct CompressorState #endif }; -static void ParseCompressionOption(int compression, CompressionAlgorithm *alg, - int *level); - /* Routines that support zlib compressed data I/O */ #ifdef HAVE_LIBZ static void InitCompressorZlib(CompressorState *cs, int level); @@ -93,57 +90,30 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF); static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs, const char *data, size_t dLen); -/* - * Interprets a numeric 'compression' value. The algorithm implied by the - * value (zlib or none at the moment), is returned in *alg, and the - * zlib compression level in *level. - */ -static void -ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level) -{ - if (compression == Z_DEFAULT_COMPRESSION || - (compression > 0 && compression <= 9)) - *alg = COMPR_ALG_LIBZ; - else if (compression == 0) - *alg = COMPR_ALG_NONE; - else - { - pg_fatal("invalid compression code: %d", compression); - *alg = COMPR_ALG_NONE; /* keep compiler quiet */ - } - - /* The level is just the passed-in value. */ - if (level) - *level = compression; -} - /* Public interface routines */ /* Allocate a new compressor */ CompressorState * -AllocateCompressor(int compression, WriteFunc writeF) +AllocateCompressor(const pg_compress_specification compression_spec, + WriteFunc writeF) { CompressorState *cs; - CompressionAlgorithm alg; - int level; - - ParseCompressionOption(compression, &alg, &level); #ifndef HAVE_LIBZ - if (alg == COMPR_ALG_LIBZ) + if (compression_spec.algorithm == PG_COMPRESSION_GZIP) pg_fatal("not built with zlib support"); #endif cs = (CompressorState *) pg_malloc0(sizeof(CompressorState)); cs->writeF = writeF; - cs->comprAlg = alg; + cs->compression_spec = compression_spec; /* * Perform compression algorithm specific initialization. */ #ifdef HAVE_LIBZ - if (alg == COMPR_ALG_LIBZ) - InitCompressorZlib(cs, level); + if (cs->compression_spec.algorithm == PG_COMPRESSION_GZIP) + InitCompressorZlib(cs, cs->compression_spec.level); #endif return cs; @@ -154,15 +124,12 @@ AllocateCompressor(int compression, WriteFunc writeF) * out with ahwrite(). */ void -ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF) +ReadDataFromArchive(ArchiveHandle *AH, pg_compress_specification compression_spec, + ReadFunc readF) { - CompressionAlgorithm alg; - - ParseCompressionOption(compression, &alg, NULL); - - if (alg == COMPR_ALG_NONE) + if (compression_spec.algorithm == PG_COMPRESSION_NONE) ReadDataFromArchiveNone(AH, readF); - if (alg == COMPR_ALG_LIBZ) + if (compression_spec.algorithm == PG_COMPRESSION_GZIP) { #ifdef HAVE_LIBZ ReadDataFromArchiveZlib(AH, readF); @@ -179,18 +146,23 @@ void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, const void *data, size_t dLen) { - switch (cs->comprAlg) + switch (cs->compression_spec.algorithm) { - case COMPR_ALG_LIBZ: + case PG_COMPRESSION_GZIP: #ifdef HAVE_LIBZ WriteDataToArchiveZlib(AH, cs, data, dLen); #else pg_fatal("not built with zlib support"); #endif break; - case COMPR_ALG_NONE: + case PG_COMPRESSION_NONE: WriteDataToArchiveNone(AH, cs, data, dLen); break; + case PG_COMPRESSION_LZ4: + /* fallthrough */ + case PG_COMPRESSION_ZSTD: + pg_fatal("invalid compression method"); + break; } } @@ -201,7 +173,7 @@ void EndCompressor(ArchiveHandle *AH, CompressorState *cs) { #ifdef HAVE_LIBZ - if (cs->comprAlg == COMPR_ALG_LIBZ) + if (cs->compression_spec.algorithm == PG_COMPRESSION_GZIP) EndCompressorZlib(AH, cs); #endif free(cs); @@ -453,20 +425,27 @@ cfopen_read(const char *path, const char *mode) { cfp *fp; + pg_compress_specification compression_spec = {0}; + #ifdef HAVE_LIBZ if (hasSuffix(path, ".gz")) - fp = cfopen(path, mode, 1); + { + compression_spec.algorithm = PG_COMPRESSION_GZIP; + fp = cfopen(path, mode, compression_spec); + } else #endif { - fp = cfopen(path, mode, 0); + compression_spec.algorithm = PG_COMPRESSION_NONE; + fp = cfopen(path, mode, compression_spec); #ifdef HAVE_LIBZ if (fp == NULL) { char *fname; fname = psprintf("%s.gz", path); - fp = cfopen(fname, mode, 1); + compression_spec.algorithm = PG_COMPRESSION_GZIP; + fp = cfopen(fname, mode, compression_spec); free_keep_errno(fname); } #endif @@ -479,26 +458,27 @@ cfopen_read(const char *path, const char *mode) * be a filemode as accepted by fopen() and gzopen() that indicates writing * ("w", "wb", "a", or "ab"). * - * If 'compression' is non-zero, a gzip compressed stream is opened, and - * 'compression' indicates the compression level used. The ".gz" suffix - * is automatically added to 'path' in that case. + * If 'compression_spec.algorithm' is GZIP, a gzip compressed stream is opened, + * and 'compression_spec.level' used. The ".gz" suffix is automatically added to + * 'path' in that case. * * On failure, return NULL with an error code in errno. */ cfp * -cfopen_write(const char *path, const char *mode, int compression) +cfopen_write(const char *path, const char *mode, + const pg_compress_specification compression_spec) { cfp *fp; - if (compression == 0) - fp = cfopen(path, mode, 0); + if (compression_spec.algorithm == PG_COMPRESSION_NONE) + fp = cfopen(path, mode, compression_spec); else { #ifdef HAVE_LIBZ char *fname; fname = psprintf("%s.gz", path); - fp = cfopen(fname, mode, compression); + fp = cfopen(fname, mode, compression_spec); free_keep_errno(fname); #else pg_fatal("not built with zlib support"); @@ -509,26 +489,27 @@ cfopen_write(const char *path, const char *mode, int compression) } /* - * Opens file 'path' in 'mode'. If 'compression' is non-zero, the file + * Opens file 'path' in 'mode'. If compression is GZIP, the file * is opened with libz gzopen(), otherwise with plain fopen(). * * On failure, return NULL with an error code in errno. */ cfp * -cfopen(const char *path, const char *mode, int compression) +cfopen(const char *path, const char *mode, + const pg_compress_specification compression_spec) { cfp *fp = pg_malloc(sizeof(cfp)); - if (compression != 0) + if (compression_spec.algorithm == PG_COMPRESSION_GZIP) { #ifdef HAVE_LIBZ - if (compression != Z_DEFAULT_COMPRESSION) + if (compression_spec.level != Z_DEFAULT_COMPRESSION) { /* user has specified a compression level, so tell zlib to use it */ char mode_compression[32]; snprintf(mode_compression, sizeof(mode_compression), "%s%d", - mode, compression); + mode, compression_spec.level); fp->compressedfp = gzopen(path, mode_compression); } else diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h index f635787692..6fad6c2cd5 100644 --- a/src/bin/pg_dump/compress_io.h +++ b/src/bin/pg_dump/compress_io.h @@ -21,12 +21,6 @@ #define ZLIB_OUT_SIZE 4096 #define ZLIB_IN_SIZE 4096 -typedef enum -{ - COMPR_ALG_NONE, - COMPR_ALG_LIBZ -} CompressionAlgorithm; - /* Prototype for callback function to WriteDataToArchive() */ typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len); @@ -46,8 +40,10 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen); /* struct definition appears in compress_io.c */ typedef struct CompressorState CompressorState; -extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF); -extern void ReadDataFromArchive(ArchiveHandle *AH, int compression, +extern CompressorState *AllocateCompressor(const pg_compress_specification compression_spec, + WriteFunc writeF); +extern void ReadDataFromArchive(ArchiveHandle *AH, + const pg_compress_specification compression_spec, ReadFunc readF); extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, const void *data, size_t dLen); @@ -56,9 +52,13 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs); typedef struct cfp cfp; -extern cfp *cfopen(const char *path, const char *mode, int compression); +extern cfp *cfopen(const char *path, const char *mode, + const pg_compress_specification compression_spec); +extern cfp *cfdopen(int fd, const char *mode, + pg_compress_specification compression_spec); extern cfp *cfopen_read(const char *path, const char *mode); -extern cfp *cfopen_write(const char *path, const char *mode, int compression); +extern cfp *cfopen_write(const char *path, const char *mode, + const pg_compress_specification compression_spec); extern int cfread(void *ptr, int size, cfp *fp); extern int cfwrite(const void *ptr, int size, cfp *fp); extern int cfgetc(cfp *fp); diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index e8b7898297..bc6b6594af 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -23,6 +23,7 @@ #ifndef PG_BACKUP_H #define PG_BACKUP_H +#include "common/compression.h" #include "fe_utils/simple_list.h" #include "libpq-fe.h" @@ -143,7 +144,8 @@ typedef struct _restoreOptions int noDataForFailedTables; int exit_on_error; - int compression; + pg_compress_specification compression_spec; /* Specification for + * compression */ int suppressDumpWarnings; /* Suppress output of WARNING entries * to stderr */ bool single_txn; @@ -303,7 +305,8 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt); /* Create a new archive */ extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + const pg_compress_specification compression_spec, + bool dosync, ArchiveMode mode, SetupWorkerPtrType setupDumpWorker); /* The --list option */ diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index f39c0fa36f..22238539fd 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -70,7 +70,8 @@ typedef struct _parallelReadyList static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + const pg_compress_specification compression_spec, + bool dosync, ArchiveMode mode, SetupWorkerPtrType setupWorkerPtr); static void _getObjectDescription(PQExpBuffer buf, const TocEntry *te); static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData); @@ -98,7 +99,8 @@ static int _discoverArchiveFormat(ArchiveHandle *AH); static int RestoringToDB(ArchiveHandle *AH); static void dump_lo_buf(ArchiveHandle *AH); static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim); -static void SetOutput(ArchiveHandle *AH, const char *filename, int compression); +static void SetOutput(ArchiveHandle *AH, const char *filename, + const pg_compress_specification compression_spec); static OutputContext SaveOutput(ArchiveHandle *AH); static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext); @@ -239,12 +241,13 @@ setupRestoreWorker(Archive *AHX) /* Public */ Archive * CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + const pg_compress_specification compression_spec, + bool dosync, ArchiveMode mode, SetupWorkerPtrType setupDumpWorker) { - ArchiveHandle *AH = _allocAH(FileSpec, fmt, compression, dosync, - mode, setupDumpWorker); + ArchiveHandle *AH = _allocAH(FileSpec, fmt, compression_spec, + dosync, mode, setupDumpWorker); return (Archive *) AH; } @@ -254,7 +257,12 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt, Archive * OpenArchive(const char *FileSpec, const ArchiveFormat fmt) { - ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker); + ArchiveHandle *AH; + pg_compress_specification compression_spec = {0}; + + compression_spec.algorithm = PG_COMPRESSION_NONE; + AH = _allocAH(FileSpec, fmt, compression_spec, true, + archModeRead, setupRestoreWorker); return (Archive *) AH; } @@ -384,7 +392,8 @@ RestoreArchive(Archive *AHX) * Make sure we won't need (de)compression we haven't got */ #ifndef HAVE_LIBZ - if (AH->compression != 0 && AH->PrintTocDataPtr != NULL) + if (AH->compression_spec.algorithm == PG_COMPRESSION_GZIP && + AH->PrintTocDataPtr != NULL) { for (te = AH->toc->next; te != AH->toc; te = te->next) { @@ -459,8 +468,8 @@ RestoreArchive(Archive *AHX) * Setup the output file if necessary. */ sav = SaveOutput(AH); - if (ropt->filename || ropt->compression) - SetOutput(AH, ropt->filename, ropt->compression); + if (ropt->filename || ropt->compression_spec.algorithm != PG_COMPRESSION_NONE) + SetOutput(AH, ropt->filename, ropt->compression_spec); ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n"); @@ -739,7 +748,7 @@ RestoreArchive(Archive *AHX) */ AH->stage = STAGE_FINALIZING; - if (ropt->filename || ropt->compression) + if (ropt->filename || ropt->compression_spec.algorithm != PG_COMPRESSION_NONE) RestoreOutput(AH, sav); if (ropt->useDB) @@ -969,6 +978,8 @@ NewRestoreOptions(void) opts->format = archUnknown; opts->cparams.promptPassword = TRI_DEFAULT; opts->dumpSections = DUMP_UNSECTIONED; + opts->compression_spec.algorithm = PG_COMPRESSION_NONE; + opts->compression_spec.level = 0; return opts; } @@ -1115,23 +1126,28 @@ PrintTOCSummary(Archive *AHX) ArchiveHandle *AH = (ArchiveHandle *) AHX; RestoreOptions *ropt = AH->public.ropt; TocEntry *te; + pg_compress_specification out_compression_spec = {0}; teSection curSection; OutputContext sav; const char *fmtName; char stamp_str[64]; + /* TOC is always uncompressed */ + out_compression_spec.algorithm = PG_COMPRESSION_NONE; + sav = SaveOutput(AH); if (ropt->filename) - SetOutput(AH, ropt->filename, 0 /* no compression */ ); + SetOutput(AH, ropt->filename, out_compression_spec); if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT, localtime(&AH->createDate)) == 0) strcpy(stamp_str, "[unknown]"); ahprintf(AH, ";\n; Archive created at %s\n", stamp_str); - ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n", + ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %s\n", sanitize_line(AH->archdbname, false), - AH->tocCount, AH->compression); + AH->tocCount, + get_compress_algorithm_name(AH->compression_spec.algorithm)); switch (AH->format) { @@ -1485,7 +1501,8 @@ archprintf(Archive *AH, const char *fmt,...) *******************************/ static void -SetOutput(ArchiveHandle *AH, const char *filename, int compression) +SetOutput(ArchiveHandle *AH, const char *filename, + const pg_compress_specification compression_spec) { int fn; @@ -1508,12 +1525,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression) /* If compression explicitly requested, use gzopen */ #ifdef HAVE_LIBZ - if (compression != 0) + if (compression_spec.algorithm == PG_COMPRESSION_GZIP) { char fmode[14]; /* Don't use PG_BINARY_x since this is zlib */ - sprintf(fmode, "wb%d", compression); + sprintf(fmode, "wb%d", compression_spec.level); if (fn >= 0) AH->OF = gzdopen(dup(fn), fmode); else @@ -2198,7 +2215,8 @@ _discoverArchiveFormat(ArchiveHandle *AH) */ static ArchiveHandle * _allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + const pg_compress_specification compression_spec, + bool dosync, ArchiveMode mode, SetupWorkerPtrType setupWorkerPtr) { ArchiveHandle *AH; @@ -2249,7 +2267,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, AH->toc->prev = AH->toc; AH->mode = mode; - AH->compression = compression; + AH->compression_spec = compression_spec; AH->dosync = dosync; memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse)); @@ -2264,7 +2282,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, * Force stdin/stdout into binary mode if that is what we are using. */ #ifdef WIN32 - if ((fmt != archNull || compression != 0) && + if ((fmt != archNull || compression_spec.algorithm != PG_COMPRESSION_NONE) && (AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0)) { if (mode == archModeWrite) @@ -3669,7 +3687,12 @@ WriteHead(ArchiveHandle *AH) AH->WriteBytePtr(AH, AH->intSize); AH->WriteBytePtr(AH, AH->offSize); AH->WriteBytePtr(AH, AH->format); - WriteInt(AH, AH->compression); + /* + * For now the compression type is implied by the level. This will need + * to change once support for more compression algorithms is added, + * requiring a format bump. + */ + WriteInt(AH, AH->compression_spec.level); crtm = *localtime(&AH->createDate); WriteInt(AH, crtm.tm_sec); WriteInt(AH, crtm.tm_min); @@ -3740,18 +3763,23 @@ ReadHead(ArchiveHandle *AH) pg_fatal("expected format (%d) differs from format found in file (%d)", AH->format, fmt); + /* Guess the compression method based on the level */ + AH->compression_spec.algorithm = PG_COMPRESSION_NONE; if (AH->version >= K_VERS_1_2) { if (AH->version < K_VERS_1_4) - AH->compression = AH->ReadBytePtr(AH); + AH->compression_spec.level = AH->ReadBytePtr(AH); else - AH->compression = ReadInt(AH); + AH->compression_spec.level = ReadInt(AH); + + if (AH->compression_spec.level != 0) + AH->compression_spec.algorithm = PG_COMPRESSION_GZIP; } else - AH->compression = Z_DEFAULT_COMPRESSION; + AH->compression_spec.algorithm = PG_COMPRESSION_GZIP; #ifndef HAVE_LIBZ - if (AH->compression != 0) + if (AH->compression_spec.algorithm == PG_COMPRESSION_GZIP) pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available"); #endif diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h index 42687c4ec8..a9560c6045 100644 --- a/src/bin/pg_dump/pg_backup_archiver.h +++ b/src/bin/pg_dump/pg_backup_archiver.h @@ -331,14 +331,8 @@ struct _archiveHandle DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */ struct _tocEntry *currToc; /* Used when dumping data */ - int compression; /*--------- - * Compression requested on open(). - * Possible values for compression: - * -1 Z_DEFAULT_COMPRESSION - * 0 COMPRESSION_NONE - * 1-9 levels for gzip compression - *--------- - */ + pg_compress_specification compression_spec; /* Requested specification for + * compression */ bool dosync; /* data requested to be synced on sight */ ArchiveMode mode; /* File mode - r or w */ void *formatData; /* Header data specific to file format */ diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c index a0a55a1edd..f413d01fcb 100644 --- a/src/bin/pg_dump/pg_backup_custom.c +++ b/src/bin/pg_dump/pg_backup_custom.c @@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te) _WriteByte(AH, BLK_DATA); /* Block type */ WriteInt(AH, te->dumpId); /* For sanity check */ - ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc); + ctx->cs = AllocateCompressor(AH->compression_spec, _CustomWriteFunc); } /* @@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) WriteInt(AH, oid); - ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc); + ctx->cs = AllocateCompressor(AH->compression_spec, _CustomWriteFunc); } /* @@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te) static void _PrintData(ArchiveHandle *AH) { - ReadDataFromArchive(AH, AH->compression, _CustomReadFunc); + ReadDataFromArchive(AH, AH->compression_spec, _CustomReadFunc); } static void diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c index 798182b6f7..53ef8db728 100644 --- a/src/bin/pg_dump/pg_backup_directory.c +++ b/src/bin/pg_dump/pg_backup_directory.c @@ -327,7 +327,8 @@ _StartData(ArchiveHandle *AH, TocEntry *te) setFilePath(AH, fname, tctx->filename); - ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression); + ctx->dataFH = cfopen_write(fname, PG_BINARY_W, + AH->compression_spec); if (ctx->dataFH == NULL) pg_fatal("could not open output file \"%s\": %m", fname); } @@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH) if (AH->mode == archModeWrite) { cfp *tocFH; + pg_compress_specification compression_spec = {0}; char fname[MAXPGPATH]; setFilePath(AH, fname, "toc.dat"); @@ -581,7 +583,8 @@ _CloseArchive(ArchiveHandle *AH) ctx->pstate = ParallelBackupStart(AH); /* The TOC is always created uncompressed */ - tocFH = cfopen_write(fname, PG_BINARY_W, 0); + compression_spec.algorithm = PG_COMPRESSION_NONE; + tocFH = cfopen_write(fname, PG_BINARY_W, compression_spec); if (tocFH == NULL) pg_fatal("could not open output file \"%s\": %m", fname); ctx->dataFH = tocFH; @@ -639,12 +642,14 @@ static void _StartBlobs(ArchiveHandle *AH, TocEntry *te) { lclContext *ctx = (lclContext *) AH->formatData; + pg_compress_specification compression_spec = {0}; char fname[MAXPGPATH]; setFilePath(AH, fname, "blobs.toc"); /* The blob TOC file is never compressed */ - ctx->blobsTocFH = cfopen_write(fname, "ab", 0); + compression_spec.algorithm = PG_COMPRESSION_NONE; + ctx->blobsTocFH = cfopen_write(fname, "ab", compression_spec); if (ctx->blobsTocFH == NULL) pg_fatal("could not open output file \"%s\": %m", fname); } @@ -662,7 +667,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid); - ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression); + ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression_spec); if (ctx->dataFH == NULL) pg_fatal("could not open output file \"%s\": %m", fname); diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c index 402b93c610..99f3f5bcae 100644 --- a/src/bin/pg_dump/pg_backup_tar.c +++ b/src/bin/pg_dump/pg_backup_tar.c @@ -194,7 +194,7 @@ InitArchiveFmt_Tar(ArchiveHandle *AH) * possible since gzdopen uses buffered IO which totally screws file * positioning. */ - if (AH->compression != 0) + if (AH->compression_spec.algorithm != PG_COMPRESSION_NONE) pg_fatal("compression is not supported by tar archive format"); } else @@ -328,7 +328,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) } } - if (AH->compression == 0) + if (AH->compression_spec.algorithm == PG_COMPRESSION_NONE) tm->nFH = ctx->tarFH; else pg_fatal("compression is not supported by tar archive format"); @@ -383,7 +383,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) umask(old_umask); - if (AH->compression == 0) + if (AH->compression_spec.algorithm == PG_COMPRESSION_NONE) tm->nFH = tm->tmpFH; else pg_fatal("compression is not supported by tar archive format"); @@ -401,7 +401,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) static void tarClose(ArchiveHandle *AH, TAR_MEMBER *th) { - if (AH->compression != 0) + if (AH->compression_spec.algorithm != PG_COMPRESSION_NONE) pg_fatal("compression is not supported by tar archive format"); if (th->mode == 'w') @@ -800,7 +800,6 @@ _CloseArchive(ArchiveHandle *AH) memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions)); ropt->filename = NULL; ropt->dropSchema = 1; - ropt->compression = 0; ropt->superuser = NULL; ropt->suppressDumpWarnings = true; @@ -888,7 +887,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) if (oid == 0) pg_fatal("invalid OID for large object (%u)", oid); - if (AH->compression != 0) + if (AH->compression_spec.algorithm != PG_COMPRESSION_NONE) pg_fatal("compression is not supported by tar archive format"); sprintf(fname, "blob_%u.dat", oid); diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index da427f4d4a..510555ef21 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -105,6 +105,8 @@ static Oid g_last_builtin_oid; /* value of the last builtin oid */ /* The specified names/patterns should to match at least one entity */ static int strict_names = 0; +static pg_compress_algorithm compression_algorithm = PG_COMPRESSION_NONE; + /* * Object inclusion/exclusion lists * @@ -340,10 +342,13 @@ main(int argc, char **argv) const char *dumpsnapshot = NULL; char *use_role = NULL; int numWorkers = 1; - int compressLevel = -1; int plainText = 0; ArchiveFormat archiveFormat = archUnknown; ArchiveMode archiveMode; + pg_compress_specification compression_spec = {0}; + char *compression_detail = NULL; + char *compression_algorithm_str = "none"; + char *error_detail = NULL; static DumpOptions dopt; @@ -561,10 +566,9 @@ main(int argc, char **argv) dopt.aclsSkip = true; break; - case 'Z': /* Compression Level */ - if (!option_parse_int(optarg, "-Z/--compress", 0, 9, - &compressLevel)) - exit_nicely(1); + case 'Z': /* Compression */ + parse_compress_options(optarg, &compression_algorithm_str, + &compression_detail); break; case 0: @@ -687,22 +691,33 @@ main(int argc, char **argv) if (archiveFormat == archNull) plainText = 1; - /* Custom and directory formats are compressed by default, others not */ - if (compressLevel == -1) - { -#ifdef HAVE_LIBZ - if (archiveFormat == archCustom || archiveFormat == archDirectory) - compressLevel = Z_DEFAULT_COMPRESSION; - else -#endif - compressLevel = 0; - } + /* + * Compression options + */ + if (!parse_compress_algorithm(compression_algorithm_str, + &compression_algorithm)) + pg_fatal("unrecognized compression algorithm: \"%s\"", + compression_algorithm_str); -#ifndef HAVE_LIBZ - if (compressLevel != 0) - pg_log_warning("requested compression not available in this installation -- archive will be uncompressed"); - compressLevel = 0; -#endif + parse_compress_specification(compression_algorithm, compression_detail, + &compression_spec); + error_detail = validate_compress_specification(&compression_spec); + if (error_detail != NULL) + pg_fatal("invalid compression specification: %s", + error_detail); + + switch (compression_algorithm) + { + case PG_COMPRESSION_NONE: + case PG_COMPRESSION_GZIP: + break; + case PG_COMPRESSION_ZSTD: + pg_fatal("compression with %s is not yet supported", "ZSTD"); + break; + case PG_COMPRESSION_LZ4: + pg_fatal("compression with %s is not yet supported", "LZ4"); + break; + } /* * If emitting an archive format, we always want to emit a DATABASE item, @@ -716,8 +731,8 @@ main(int argc, char **argv) pg_fatal("parallel backup only supported by the directory format"); /* Open the output file */ - fout = CreateArchive(filename, archiveFormat, compressLevel, dosync, - archiveMode, setupDumpWorker); + fout = CreateArchive(filename, archiveFormat, compression_spec, + dosync, archiveMode, setupDumpWorker); /* Make dump options accessible right away */ SetArchiveOptions(fout, &dopt, NULL); @@ -948,10 +963,7 @@ main(int argc, char **argv) ropt->sequence_data = dopt.sequence_data; ropt->binary_upgrade = dopt.binary_upgrade; - if (compressLevel == -1) - ropt->compression = 0; - else - ropt->compression = compressLevel; + ropt->compression_spec = compression_spec; ropt->suppressDumpWarnings = true; /* We've already shown them */ @@ -998,7 +1010,8 @@ help(const char *progname) printf(_(" -j, --jobs=NUM use this many parallel jobs to dump\n")); printf(_(" -v, --verbose verbose mode\n")); printf(_(" -V, --version output version information, then exit\n")); - printf(_(" -Z, --compress=0-9 compression level for compressed formats\n")); + printf(_(" -Z, --compress=METHOD[:LEVEL]\n" + " compress as specified\n")); printf(_(" --lock-wait-timeout=TIMEOUT fail after waiting TIMEOUT for a table lock\n")); printf(_(" --no-sync do not wait for changes to be written safely to disk\n")); printf(_(" -?, --help show this help, then exit\n")); diff --git a/src/bin/pg_dump/t/001_basic.pl b/src/bin/pg_dump/t/001_basic.pl index a583c8a6d2..c8bc02126d 100644 --- a/src/bin/pg_dump/t/001_basic.pl +++ b/src/bin/pg_dump/t/001_basic.pl @@ -121,24 +121,46 @@ command_fails_like( 'pg_restore: cannot specify both --single-transaction and multiple jobs'); command_fails_like( - [ 'pg_dump', '-Z', '-1' ], - qr/\Qpg_dump: error: -Z\/--compress must be in range 0..9\E/, - 'pg_dump: -Z/--compress must be in range'); + [ 'pg_dump', '--compress', 'garbage' ], + qr/\Qpg_dump: error: unrecognized compression algorithm/, + 'pg_dump: invalid --compress'); + +command_fails_like( + [ 'pg_dump', '--compress', 'none:1' ], + qr/\Qpg_dump: error: invalid compression specification: compression algorithm "none" does not accept a compression level\E/, + 'pg_dump: invalid compression specification: compression algorithm "none" does not accept a compression level' +); + if (check_pg_config("#define HAVE_LIBZ 1")) { + command_fails_like( + [ 'pg_dump', '-Z', '15' ], + qr/\Qpg_dump: error: invalid compression specification: compression algorithm "gzip" expects a compression level between 1 and 9 (default at -1)\E/, + 'pg_dump: invalid compression specification: must be in range'); + command_fails_like( [ 'pg_dump', '--compress', '1', '--format', 'tar' ], qr/\Qpg_dump: error: compression is not supported by tar archive format\E/, 'pg_dump: compression is not supported by tar archive format'); + + command_fails_like( + [ 'pg_dump', '-Z', 'gzip:nonInt' ], + qr/\Qpg_dump: error: invalid compression specification: unrecognized compression option: "nonInt"\E/, + 'pg_dump: invalid compression specification: must be an integer'); } else { # --jobs > 1 forces an error with tar format. command_fails_like( - [ 'pg_dump', '--compress', '1', '--format', 'tar', '-j3' ], - qr/\Qpg_dump: warning: requested compression not available in this installation -- archive will be uncompressed\E/, - 'pg_dump: warning: compression not available in this installation'); + [ 'pg_dump', '--format', 'tar', '-j3' ], + qr/\Qpg_dump: error: parallel backup only supported by the directory format\E/, + 'pg_dump: warning: parallel backup not supported by tar format'); + + command_fails_like( + [ 'pg_dump', '-Z', 'gzip:nonInt', '--format', 'tar', '-j2' ], + qr/\Qpg_dump: error: invalid compression specification: unrecognized compression option\E/, + 'pg_dump: invalid compression specification: must be an integer'); } command_fails_like( diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl index fe53ed0f89..709db0986d 100644 --- a/src/bin/pg_dump/t/002_pg_dump.pl +++ b/src/bin/pg_dump/t/002_pg_dump.pl @@ -87,7 +87,7 @@ my %pgdump_runs = ( compile_option => 'gzip', dump_cmd => [ 'pg_dump', '--jobs=2', - '--format=directory', '--compress=1', + '--format=directory', '--compress=gzip:1', "--file=$tempdir/compression_gzip_dir", 'postgres', ], # Give coverage for manually compressed blob.toc files during @@ -200,6 +200,7 @@ my %pgdump_runs = ( # Do not use --no-sync to give test coverage for data sync. defaults_custom_format => { test_key => 'defaults', + compile_option => 'gzip', dump_cmd => [ 'pg_dump', '-Fc', '-Z6', "--file=$tempdir/defaults_custom_format.dump", 'postgres', diff --git a/src/test/modules/test_pg_dump/t/001_base.pl b/src/test/modules/test_pg_dump/t/001_base.pl index f5da6bf46d..19577ce0ea 100644 --- a/src/test/modules/test_pg_dump/t/001_base.pl +++ b/src/test/modules/test_pg_dump/t/001_base.pl @@ -20,6 +20,10 @@ my $tempdir = PostgreSQL::Test::Utils::tempdir; # to define how each test should (or shouldn't) treat a result # from a given run. # +# compile_option indicates if the commands run depend on a compilation +# option, if any. This can be used to control if tests should be +# skipped when a build dependency is not satisfied. +# # test_key indicates that a given run should simply use the same # set of like/unlike tests as another run, and which run that is. # @@ -90,6 +94,7 @@ my %pgdump_runs = ( }, defaults_custom_format => { test_key => 'defaults', + compile_option => 'gzip', dump_cmd => [ 'pg_dump', '--no-sync', '-Fc', '-Z6', "--file=$tempdir/defaults_custom_format.dump", 'postgres', @@ -749,6 +754,8 @@ $node->start; my $port = $node->port; +my $supports_gzip = check_pg_config("#define HAVE_LIBZ 1"); + ######################################### # Set up schemas, tables, etc, to be dumped. @@ -792,6 +799,15 @@ foreach my $run (sort keys %pgdump_runs) my $test_key = $run; + # Skip command-level tests for gzip if there is no support for it. + if ( defined($pgdump_runs{$run}->{compile_option}) + && $pgdump_runs{$run}->{compile_option} eq 'gzip' + && !$supports_gzip) + { + note "$run: skipped due to no gzip support"; + next; + } + $node->command_ok(\@{ $pgdump_runs{$run}->{dump_cmd} }, "$run: pg_dump runs"); diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml index 8b9d9f4cad..2a015908f0 100644 --- a/doc/src/sgml/ref/pg_dump.sgml +++ b/doc/src/sgml/ref/pg_dump.sgml @@ -644,17 +644,35 @@ PostgreSQL documentation - - + + [:level] + + [:level] - Specify the compression level to use. Zero means no compression. + Specify the compression method and/or the compression level to use. + The compression method can be set to gzip or + none for no compression. A compression level can + be optionally specified, by appending the level number after a colon + (:). + + + If no compression level is specified, the default compression + level will be used. If only a level is specified without mentioning + an algorithm, gzip compression will be used if + the level is greater than 0, and no compression + will be used if the level is 0. + + + For the custom and directory archive formats, this specifies compression of - individual table-data segments, and the default is to compress - at a moderate level. - For plain text output, setting a nonzero compression level causes - the entire output file to be compressed, as though it had been - fed through gzip; but the default is not to compress. + individual table-data segments, and the default is to compress using + gzip at a moderate level. For plain text output, + setting a nonzero compression level causes the entire output file to be compressed, + as though it had been fed through gzip; but the default + is not to compress. + + The tar archive format currently does not support compression at all. diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 2f5802195d..58daeca831 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -428,7 +428,6 @@ CompiledExprState CompositeIOData CompositeTypeStmt CompoundAffixFlag -CompressionAlgorithm CompressionLocation CompressorState ComputeXidHorizonsResult -- 2.38.1