From 7d2edf176b1ba4c70253dbeec0e878b9ed515349 Mon Sep 17 00:00:00 2001 From: Georgios Kokolatos Date: Wed, 30 Mar 2022 15:03:53 +0000 Subject: [PATCH v4 3/4] Prepare pg_dump for additional compression methods This commmit does the heavy lifting required for additional compression methods. Commit bf9aa490db introduced cfp in compress_io.{c,h} with the intent of unifying compression related code and allow for the introduction of additional archive formats. However, pg_backup_archiver.c was not using that API. This commit teaches pg_backup_archiver.c about cfp and is using it through out. Furthermore, compression was chosen based on the value of the level passed as an argument during the invocation of pg_dump or some hardcoded defaults. This does not scale for more than one compression methods. Now the method used for compression can be explicitly requested during command invocation, or set during hardcoded defaults. Then it is stored in the relevant structs and passed in the relevant functions, along side compression level which has lost it's special meaning. The method for compression is not yet stored in the actual archive. This is done in the next commit which does introduce a new method. The previously named CompressionAlgorithm enum is changed for CompressionMethod so that it matches better similar variables found through out the code base. In a fashion similar to the binary for pg_basebackup, the method for compression is passed using the already existing -Z/--compress parameter of pg_dump. The legacy format and behaviour is maintained. Additionally, the user can explicitly pass a requested method and optionaly the level to be used after a semicolon, e.g. --compress=gzip:6 --- doc/src/sgml/ref/pg_dump.sgml | 30 +- src/bin/pg_dump/compress_io.c | 416 ++++++++++++++++---------- src/bin/pg_dump/compress_io.h | 32 +- src/bin/pg_dump/pg_backup.h | 14 +- src/bin/pg_dump/pg_backup_archiver.c | 171 +++++------ src/bin/pg_dump/pg_backup_archiver.h | 46 +-- src/bin/pg_dump/pg_backup_custom.c | 11 +- src/bin/pg_dump/pg_backup_directory.c | 12 +- src/bin/pg_dump/pg_backup_tar.c | 12 +- src/bin/pg_dump/pg_dump.c | 155 ++++++++-- src/bin/pg_dump/t/001_basic.pl | 14 +- src/bin/pg_dump/t/002_pg_dump.pl | 45 ++- src/tools/pgindent/typedefs.list | 2 +- 13 files changed, 603 insertions(+), 357 deletions(-) diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml index 2f0042fd96..992b7312df 100644 --- a/doc/src/sgml/ref/pg_dump.sgml +++ b/doc/src/sgml/ref/pg_dump.sgml @@ -644,17 +644,31 @@ PostgreSQL documentation - - + + [:level] + + [:level] - Specify the compression level to use. Zero means no compression. + Specify the compression method and/or the compression level to use. + The compression method can be set to gzip or + none for no compression. A compression level can + be optionally specified, by appending the level number after a colon + (:). If no level is specified, the default compression + level will be used for the specified method. If only a level is + specified without mentioning a method, gzip compression + will be used. + + + For the custom and directory archive formats, this specifies compression of - individual table-data segments, and the default is to compress - at a moderate level. - For plain text output, setting a nonzero compression level causes - the entire output file to be compressed, as though it had been - fed through gzip; but the default is not to compress. + individual table-data segments, and the default is to compress using + gzip at a moderate level. For plain text output, + setting a nonzero compression level causes the entire output file to be compressed, + as though it had been fed through gzip; but the default + is not to compress. + + The tar archive format currently does not support compression at all. diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c index 9077fdb74d..630f9e4b18 100644 --- a/src/bin/pg_dump/compress_io.c +++ b/src/bin/pg_dump/compress_io.c @@ -64,7 +64,7 @@ /* typedef appears in compress_io.h */ struct CompressorState { - CompressionAlgorithm comprAlg; + CompressionMethod compressionMethod; WriteFunc writeF; #ifdef HAVE_LIBZ @@ -74,9 +74,6 @@ struct CompressorState #endif }; -static void ParseCompressionOption(int compression, CompressionAlgorithm *alg, - int *level); - /* Routines that support zlib compressed data I/O */ #ifdef HAVE_LIBZ static void InitCompressorZlib(CompressorState *cs, int level); @@ -93,57 +90,30 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF); static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs, const char *data, size_t dLen); -/* - * Interprets a numeric 'compression' value. The algorithm implied by the - * value (zlib or none at the moment), is returned in *alg, and the - * zlib compression level in *level. - */ -static void -ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level) -{ - if (compression == Z_DEFAULT_COMPRESSION || - (compression > 0 && compression <= 9)) - *alg = COMPR_ALG_LIBZ; - else if (compression == 0) - *alg = COMPR_ALG_NONE; - else - { - fatal("invalid compression code: %d", compression); - *alg = COMPR_ALG_NONE; /* keep compiler quiet */ - } - - /* The level is just the passed-in value. */ - if (level) - *level = compression; -} - /* Public interface routines */ /* Allocate a new compressor */ CompressorState * -AllocateCompressor(int compression, WriteFunc writeF) +AllocateCompressor(CompressionMethod compressionMethod, + int compressionLevel, WriteFunc writeF) { CompressorState *cs; - CompressionAlgorithm alg; - int level; - - ParseCompressionOption(compression, &alg, &level); #ifndef HAVE_LIBZ - if (alg == COMPR_ALG_LIBZ) + if (compressionMethod == COMPRESSION_GZIP) fatal("not built with zlib support"); #endif cs = (CompressorState *) pg_malloc0(sizeof(CompressorState)); cs->writeF = writeF; - cs->comprAlg = alg; + cs->compressionMethod = compressionMethod; /* * Perform compression algorithm specific initialization. */ #ifdef HAVE_LIBZ - if (alg == COMPR_ALG_LIBZ) - InitCompressorZlib(cs, level); + if (compressionMethod == COMPRESSION_GZIP) + InitCompressorZlib(cs, compressionLevel); #endif return cs; @@ -154,21 +124,24 @@ AllocateCompressor(int compression, WriteFunc writeF) * out with ahwrite(). */ void -ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF) +ReadDataFromArchive(ArchiveHandle *AH, CompressionMethod compressionMethod, + int compressionLevel, ReadFunc readF) { - CompressionAlgorithm alg; - - ParseCompressionOption(compression, &alg, NULL); - - if (alg == COMPR_ALG_NONE) - ReadDataFromArchiveNone(AH, readF); - if (alg == COMPR_ALG_LIBZ) + switch (compressionMethod) { + case COMPRESSION_NONE: + ReadDataFromArchiveNone(AH, readF); + break; + case COMPRESSION_GZIP: #ifdef HAVE_LIBZ - ReadDataFromArchiveZlib(AH, readF); + ReadDataFromArchiveZlib(AH, readF); #else - fatal("not built with zlib support"); + fatal("not built with zlib support"); #endif + break; + default: + fatal("invalid compression method"); + break; } } @@ -179,18 +152,21 @@ void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, const void *data, size_t dLen) { - switch (cs->comprAlg) + switch (cs->compressionMethod) { - case COMPR_ALG_LIBZ: + case COMPRESSION_GZIP: #ifdef HAVE_LIBZ WriteDataToArchiveZlib(AH, cs, data, dLen); #else fatal("not built with zlib support"); #endif break; - case COMPR_ALG_NONE: + case COMPRESSION_NONE: WriteDataToArchiveNone(AH, cs, data, dLen); break; + default: + fatal("invalid compression method"); + break; } } @@ -200,11 +176,23 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, void EndCompressor(ArchiveHandle *AH, CompressorState *cs) { + switch (cs->compressionMethod) + { + case COMPRESSION_GZIP: #ifdef HAVE_LIBZ - if (cs->comprAlg == COMPR_ALG_LIBZ) - EndCompressorZlib(AH, cs); + EndCompressorZlib(AH, cs); +#else + fatal("not built with zlib support"); #endif - free(cs); + break; + case COMPRESSION_NONE: + free(cs); + break; + + default: + fatal("invalid compression method"); + break; + } } /* Private routines, specific to each compression method. */ @@ -418,10 +406,8 @@ WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs, */ struct cfp { - FILE *uncompressedfp; -#ifdef HAVE_LIBZ - gzFile compressedfp; -#endif + CompressionMethod compressionMethod; + void *fp; }; #ifdef HAVE_LIBZ @@ -455,18 +441,18 @@ cfopen_read(const char *path, const char *mode) #ifdef HAVE_LIBZ if (hasSuffix(path, ".gz")) - fp = cfopen(path, mode, 1); + fp = cfopen(path, mode, COMPRESSION_GZIP, 0); else #endif { - fp = cfopen(path, mode, 0); + fp = cfopen(path, mode, COMPRESSION_NONE, 0); #ifdef HAVE_LIBZ if (fp == NULL) { char *fname; fname = psprintf("%s.gz", path); - fp = cfopen(fname, mode, 1); + fp = cfopen(fname, mode, COMPRESSION_GZIP, 0); free_keep_errno(fname); } #endif @@ -486,19 +472,21 @@ cfopen_read(const char *path, const char *mode) * On failure, return NULL with an error code in errno. */ cfp * -cfopen_write(const char *path, const char *mode, int compression) +cfopen_write(const char *path, const char *mode, + CompressionMethod compressionMethod, + int compressionLevel) { cfp *fp; - if (compression == 0) - fp = cfopen(path, mode, 0); + if (compressionMethod == COMPRESSION_NONE) + fp = cfopen(path, mode, compressionMethod, 0); else { #ifdef HAVE_LIBZ char *fname; fname = psprintf("%s.gz", path); - fp = cfopen(fname, mode, compression); + fp = cfopen(fname, mode, compressionMethod, compressionLevel); free_keep_errno(fname); #else fatal("not built with zlib support"); @@ -509,60 +497,94 @@ cfopen_write(const char *path, const char *mode, int compression) } /* - * Opens file 'path' in 'mode'. If 'compression' is non-zero, the file - * is opened with libz gzopen(), otherwise with plain fopen(). + * This is the workhorse for cfopen() or cfdopen(). It opens file 'path' or + * associates a stream 'fd', if 'fd' is a valid descriptor, in 'mode'. The + * descriptor is not dup'ed and it is the caller's responsibility to do so. + * The caller must verify that the 'compressionMethod' is supported by the + * current build. * * On failure, return NULL with an error code in errno. */ -cfp * -cfopen(const char *path, const char *mode, int compression) +static cfp * +cfopen_internal(const char *path, int fd, const char *mode, + CompressionMethod compressionMethod, int compressionLevel) { cfp *fp = pg_malloc(sizeof(cfp)); - if (compression != 0) + fp->compressionMethod = compressionMethod; + + switch (compressionMethod) { -#ifdef HAVE_LIBZ - if (compression != Z_DEFAULT_COMPRESSION) - { - /* user has specified a compression level, so tell zlib to use it */ - char mode_compression[32]; + case COMPRESSION_NONE: + if (fd >= 0) + fp->fp = fdopen(fd, mode); + else + fp->fp = fopen(path, mode); + if (fp->fp == NULL) + { + free_keep_errno(fp); + fp = NULL; + } - snprintf(mode_compression, sizeof(mode_compression), "%s%d", - mode, compression); - fp->compressedfp = gzopen(path, mode_compression); - } - else - { - /* don't specify a level, just use the zlib default */ - fp->compressedfp = gzopen(path, mode); - } + break; + case COMPRESSION_GZIP: +#ifdef HAVE_LIBZ + if (compressionLevel != Z_DEFAULT_COMPRESSION) + { + /* + * user has specified a compression level, so tell zlib to use + * it + */ + char mode_compression[32]; + + snprintf(mode_compression, sizeof(mode_compression), "%s%d", + mode, compressionLevel); + if (fd >= 0) + fp->fp = gzdopen(fd, mode_compression); + else + fp->fp = gzopen(path, mode_compression); + } + else + { + /* don't specify a level, just use the zlib default */ + if (fd >= 0) + fp->fp = gzdopen(fd, mode); + else + fp->fp = gzopen(path, mode); + } - fp->uncompressedfp = NULL; - if (fp->compressedfp == NULL) - { - free_keep_errno(fp); - fp = NULL; - } + if (fp->fp == NULL) + { + free_keep_errno(fp); + fp = NULL; + } #else - fatal("not built with zlib support"); -#endif - } - else - { -#ifdef HAVE_LIBZ - fp->compressedfp = NULL; + fatal("not built with zlib support"); #endif - fp->uncompressedfp = fopen(path, mode); - if (fp->uncompressedfp == NULL) - { - free_keep_errno(fp); - fp = NULL; - } + break; + default: + fatal("invalid compression method"); + break; } return fp; } +cfp * +cfopen(const char *path, const char *mode, + CompressionMethod compressionMethod, + int compressionLevel) +{ + return cfopen_internal(path, -1, mode, compressionMethod, compressionLevel); +} + +cfp * +cfdopen(int fd, const char *mode, + CompressionMethod compressionMethod, + int compressionLevel) +{ + return cfopen_internal(NULL, fd, mode, compressionMethod, compressionLevel); +} int cfread(void *ptr, int size, cfp *fp) @@ -572,38 +594,61 @@ cfread(void *ptr, int size, cfp *fp) if (size == 0) return 0; -#ifdef HAVE_LIBZ - if (fp->compressedfp) + switch (fp->compressionMethod) { - ret = gzread(fp->compressedfp, ptr, size); - if (ret != size && !gzeof(fp->compressedfp)) - { - int errnum; - const char *errmsg = gzerror(fp->compressedfp, &errnum); + case COMPRESSION_NONE: + ret = fread(ptr, 1, size, fp->fp); + if (ret != size && !feof(fp->fp)) + READ_ERROR_EXIT(fp->fp); - fatal("could not read from input file: %s", - errnum == Z_ERRNO ? strerror(errno) : errmsg); - } - } - else + break; + case COMPRESSION_GZIP: +#ifdef HAVE_LIBZ + ret = gzread(fp->fp, ptr, size); + if (ret != size && !gzeof(fp->fp)) + { + int errnum; + const char *errmsg = gzerror(fp->fp, &errnum); + + fatal("could not read from input file: %s", + errnum == Z_ERRNO ? strerror(errno) : errmsg); + } +#else + fatal("not built with zlib support"); #endif - { - ret = fread(ptr, 1, size, fp->uncompressedfp); - if (ret != size && !feof(fp->uncompressedfp)) - READ_ERROR_EXIT(fp->uncompressedfp); + break; + + default: + fatal("invalid compression method"); + break; } + return ret; } int cfwrite(const void *ptr, int size, cfp *fp) { + int ret = 0; + + switch (fp->compressionMethod) + { + case COMPRESSION_NONE: + ret = fwrite(ptr, 1, size, fp->fp); + break; + case COMPRESSION_GZIP: #ifdef HAVE_LIBZ - if (fp->compressedfp) - return gzwrite(fp->compressedfp, ptr, size); - else + ret = gzwrite(fp->fp, ptr, size); +#else + fatal("not built with zlib support"); #endif - return fwrite(ptr, 1, size, fp->uncompressedfp); + break; + default: + fatal("invalid compression method"); + break; + } + + return ret; } int @@ -611,24 +656,31 @@ cfgetc(cfp *fp) { int ret; -#ifdef HAVE_LIBZ - if (fp->compressedfp) + switch (fp->compressionMethod) { - ret = gzgetc(fp->compressedfp); - if (ret == EOF) - { - if (!gzeof(fp->compressedfp)) - fatal("could not read from input file: %s", strerror(errno)); - else - fatal("could not read from input file: end of file"); - } - } - else + case COMPRESSION_NONE: + ret = fgetc(fp->fp); + if (ret == EOF) + READ_ERROR_EXIT(fp->fp); + + break; + case COMPRESSION_GZIP: +#ifdef HAVE_LIBZ + ret = gzgetc((gzFile)fp->fp); + if (ret == EOF) + { + if (!gzeof(fp->fp)) + fatal("could not read from input file: %s", strerror(errno)); + else + fatal("could not read from input file: end of file"); + } +#else + fatal("not built with zlib support"); #endif - { - ret = fgetc(fp->uncompressedfp); - if (ret == EOF) - READ_ERROR_EXIT(fp->uncompressedfp); + break; + default: + fatal("invalid compression method"); + break; } return ret; @@ -637,65 +689,107 @@ cfgetc(cfp *fp) char * cfgets(cfp *fp, char *buf, int len) { + char *ret; + + switch (fp->compressionMethod) + { + case COMPRESSION_NONE: + ret = fgets(buf, len, fp->fp); + + break; + case COMPRESSION_GZIP: #ifdef HAVE_LIBZ - if (fp->compressedfp) - return gzgets(fp->compressedfp, buf, len); - else + ret = gzgets(fp->fp, buf, len); +#else + fatal("not built with zlib support"); #endif - return fgets(buf, len, fp->uncompressedfp); + break; + default: + fatal("invalid compression method"); + break; + } + + return ret; } int cfclose(cfp *fp) { - int result; + int ret; if (fp == NULL) { errno = EBADF; return EOF; } -#ifdef HAVE_LIBZ - if (fp->compressedfp) + + switch (fp->compressionMethod) { - result = gzclose(fp->compressedfp); - fp->compressedfp = NULL; - } - else + case COMPRESSION_NONE: + ret = fclose(fp->fp); + fp->fp = NULL; + + break; + case COMPRESSION_GZIP: +#ifdef HAVE_LIBZ + ret = gzclose(fp->fp); + fp->fp = NULL; +#else + fatal("not built with zlib support"); #endif - { - result = fclose(fp->uncompressedfp); - fp->uncompressedfp = NULL; + break; + default: + fatal("invalid compression method"); + break; } + free_keep_errno(fp); - return result; + return ret; } int cfeof(cfp *fp) { + int ret; + + switch (fp->compressionMethod) + { + case COMPRESSION_NONE: + ret = feof(fp->fp); + + break; + case COMPRESSION_GZIP: #ifdef HAVE_LIBZ - if (fp->compressedfp) - return gzeof(fp->compressedfp); - else + ret = gzeof(fp->fp); +#else + fatal("not built with zlib support"); #endif - return feof(fp->uncompressedfp); + break; + default: + fatal("invalid compression method"); + break; + } + + return ret; } const char * get_cfp_error(cfp *fp) { -#ifdef HAVE_LIBZ - if (fp->compressedfp) + if (fp->compressionMethod == COMPRESSION_GZIP) { +#ifdef HAVE_LIBZ int errnum; - const char *errmsg = gzerror(fp->compressedfp, &errnum); + const char *errmsg = gzerror(fp->fp, &errnum); if (errnum != Z_ERRNO) return errmsg; - } +#else + fatal("not built with zlib support"); #endif + } + return strerror(errno); } diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h index f635787692..b8b366616c 100644 --- a/src/bin/pg_dump/compress_io.h +++ b/src/bin/pg_dump/compress_io.h @@ -17,16 +17,17 @@ #include "pg_backup_archiver.h" +#ifdef HAVE_LIBZ +#include +#else +/* this is just the redefinition of a libz constant */ +#define Z_DEFAULT_COMPRESSION (-1) +#endif + /* Initial buffer sizes used in zlib compression. */ #define ZLIB_OUT_SIZE 4096 #define ZLIB_IN_SIZE 4096 -typedef enum -{ - COMPR_ALG_NONE, - COMPR_ALG_LIBZ -} CompressionAlgorithm; - /* Prototype for callback function to WriteDataToArchive() */ typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len); @@ -46,8 +47,12 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen); /* struct definition appears in compress_io.c */ typedef struct CompressorState CompressorState; -extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF); -extern void ReadDataFromArchive(ArchiveHandle *AH, int compression, +extern CompressorState *AllocateCompressor(CompressionMethod compressionMethod, + int compressionLevel, + WriteFunc writeF); +extern void ReadDataFromArchive(ArchiveHandle *AH, + CompressionMethod compressionMethod, + int compressionLevel, ReadFunc readF); extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, const void *data, size_t dLen); @@ -56,9 +61,16 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs); typedef struct cfp cfp; -extern cfp *cfopen(const char *path, const char *mode, int compression); +extern cfp *cfopen(const char *path, const char *mode, + CompressionMethod compressionMethod, + int compressionLevel); +extern cfp *cfdopen(int fd, const char *mode, + CompressionMethod compressionMethod, + int compressionLevel); extern cfp *cfopen_read(const char *path, const char *mode); -extern cfp *cfopen_write(const char *path, const char *mode, int compression); +extern cfp *cfopen_write(const char *path, const char *mode, + CompressionMethod compressionMethod, + int compressionLevel); extern int cfread(void *ptr, int size, cfp *fp); extern int cfwrite(const void *ptr, int size, cfp *fp); extern int cfgetc(cfp *fp); diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index fcc5f6bd05..7645d3285a 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -75,6 +75,13 @@ enum _dumpPreparedQueries NUM_PREP_QUERIES /* must be last */ }; +typedef enum _compressionMethod +{ + COMPRESSION_INVALID, + COMPRESSION_NONE, + COMPRESSION_GZIP +} CompressionMethod; + /* Parameters needed by ConnectDatabase; same for dump and restore */ typedef struct _connParams { @@ -143,7 +150,8 @@ typedef struct _restoreOptions int noDataForFailedTables; int exit_on_error; - int compression; + CompressionMethod compressionMethod; + int compressionLevel; int suppressDumpWarnings; /* Suppress output of WARNING entries * to stderr */ bool single_txn; @@ -303,7 +311,9 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt); /* Create a new archive */ extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + const CompressionMethod compressionMethod, + const int compression, + bool dosync, ArchiveMode mode, SetupWorkerPtrType setupDumpWorker); /* The --list option */ diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index d41a99d6ea..7d96446f1a 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -31,6 +31,7 @@ #endif #include "common/string.h" +#include "compress_io.h" #include "dumputils.h" #include "fe_utils/string_utils.h" #include "lib/stringinfo.h" @@ -43,13 +44,6 @@ #define TEXT_DUMP_HEADER "--\n-- PostgreSQL database dump\n--\n\n" #define TEXT_DUMPALL_HEADER "--\n-- PostgreSQL database cluster dump\n--\n\n" -/* state needed to save/restore an archive's output target */ -typedef struct _outputContext -{ - void *OF; - int gzOut; -} OutputContext; - /* * State for tracking TocEntrys that are ready to process during a parallel * restore. (This used to be a list, and we still call it that, though now @@ -70,7 +64,9 @@ typedef struct _parallelReadyList static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + const CompressionMethod compressionMethod, + const int compressionLevel, + bool dosync, ArchiveMode mode, SetupWorkerPtrType setupWorkerPtr); static void _getObjectDescription(PQExpBuffer buf, TocEntry *te); static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData); @@ -98,9 +94,11 @@ static int _discoverArchiveFormat(ArchiveHandle *AH); static int RestoringToDB(ArchiveHandle *AH); static void dump_lo_buf(ArchiveHandle *AH); static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim); -static void SetOutput(ArchiveHandle *AH, const char *filename, int compression); -static OutputContext SaveOutput(ArchiveHandle *AH); -static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext); +static void SetOutput(ArchiveHandle *AH, const char *filename, + CompressionMethod compressionMethod, + int compressionLevel); +static cfp *SaveOutput(ArchiveHandle *AH); +static void RestoreOutput(ArchiveHandle *AH, cfp *savedOutput); static int restore_toc_entry(ArchiveHandle *AH, TocEntry *te, bool is_parallel); static void restore_toc_entries_prefork(ArchiveHandle *AH, @@ -239,12 +237,15 @@ setupRestoreWorker(Archive *AHX) /* Public */ Archive * CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + const CompressionMethod compressionMethod, + const int compressionLevel, + bool dosync, ArchiveMode mode, SetupWorkerPtrType setupDumpWorker) { - ArchiveHandle *AH = _allocAH(FileSpec, fmt, compression, dosync, - mode, setupDumpWorker); + ArchiveHandle *AH = _allocAH(FileSpec, fmt, + compressionMethod, compressionLevel, + dosync, mode, setupDumpWorker); return (Archive *) AH; } @@ -254,7 +255,8 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt, Archive * OpenArchive(const char *FileSpec, const ArchiveFormat fmt) { - ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker); + ArchiveHandle *AH = _allocAH(FileSpec, fmt, COMPRESSION_NONE, 0, true, + archModeRead, setupRestoreWorker); return (Archive *) AH; } @@ -269,11 +271,8 @@ CloseArchive(Archive *AHX) AH->ClosePtr(AH); /* Close the output */ - errno = 0; /* in case gzclose() doesn't set it */ - if (AH->gzOut) - res = GZCLOSE(AH->OF); - else if (AH->OF != stdout) - res = fclose(AH->OF); + errno = 0; + res = cfclose(AH->OF); if (res != 0) fatal("could not close output file: %m"); @@ -355,7 +354,7 @@ RestoreArchive(Archive *AHX) RestoreOptions *ropt = AH->public.ropt; bool parallel_mode; TocEntry *te; - OutputContext sav; + cfp *sav; AH->stage = STAGE_INITIALIZING; @@ -384,7 +383,8 @@ RestoreArchive(Archive *AHX) * Make sure we won't need (de)compression we haven't got */ #ifndef HAVE_LIBZ - if (AH->compression != 0 && AH->PrintTocDataPtr != NULL) + if (AH->compressionMethod == COMPRESSION_GZIP && + AH->PrintTocDataPtr != NULL) { for (te = AH->toc->next; te != AH->toc; te = te->next) { @@ -459,8 +459,9 @@ RestoreArchive(Archive *AHX) * Setup the output file if necessary. */ sav = SaveOutput(AH); - if (ropt->filename || ropt->compression) - SetOutput(AH, ropt->filename, ropt->compression); + if (ropt->filename || ropt->compressionMethod != COMPRESSION_NONE) + SetOutput(AH, ropt->filename, + ropt->compressionMethod, ropt->compressionLevel); ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n"); @@ -740,7 +741,7 @@ RestoreArchive(Archive *AHX) */ AH->stage = STAGE_FINALIZING; - if (ropt->filename || ropt->compression) + if (ropt->filename || ropt->compressionMethod != COMPRESSION_NONE) RestoreOutput(AH, sav); if (ropt->useDB) @@ -970,6 +971,7 @@ NewRestoreOptions(void) opts->format = archUnknown; opts->cparams.promptPassword = TRI_DEFAULT; opts->dumpSections = DUMP_UNSECTIONED; + opts->compressionMethod = COMPRESSION_NONE; return opts; } @@ -1117,13 +1119,13 @@ PrintTOCSummary(Archive *AHX) RestoreOptions *ropt = AH->public.ropt; TocEntry *te; teSection curSection; - OutputContext sav; + cfp *sav; const char *fmtName; char stamp_str[64]; sav = SaveOutput(AH); if (ropt->filename) - SetOutput(AH, ropt->filename, 0 /* no compression */ ); + SetOutput(AH, ropt->filename, COMPRESSION_NONE, 0); if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT, localtime(&AH->createDate)) == 0) @@ -1132,7 +1134,7 @@ PrintTOCSummary(Archive *AHX) ahprintf(AH, ";\n; Archive created at %s\n", stamp_str); ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n", sanitize_line(AH->archdbname, false), - AH->tocCount, AH->compression); + AH->tocCount, AH->compressionLevel); switch (AH->format) { @@ -1486,60 +1488,35 @@ archprintf(Archive *AH, const char *fmt,...) *******************************/ static void -SetOutput(ArchiveHandle *AH, const char *filename, int compression) +SetOutput(ArchiveHandle *AH, const char *filename, + CompressionMethod compressionMethod, int compressionLevel) { - int fn; + const char *mode; + int fn = -1; if (filename) { if (strcmp(filename, "-") == 0) fn = fileno(stdout); - else - fn = -1; } else if (AH->FH) fn = fileno(AH->FH); else if (AH->fSpec) { - fn = -1; filename = AH->fSpec; } else fn = fileno(stdout); - /* If compression explicitly requested, use gzopen */ -#ifdef HAVE_LIBZ - if (compression != 0) - { - char fmode[14]; + if (AH->mode == archModeAppend) + mode = PG_BINARY_A; + else + mode = PG_BINARY_W; - /* Don't use PG_BINARY_x since this is zlib */ - sprintf(fmode, "wb%d", compression); - if (fn >= 0) - AH->OF = gzdopen(dup(fn), fmode); - else - AH->OF = gzopen(filename, fmode); - AH->gzOut = 1; - } + if (fn >= 0) + AH->OF = cfdopen(dup(fn), mode, compressionMethod, compressionLevel); else -#endif - { /* Use fopen */ - if (AH->mode == archModeAppend) - { - if (fn >= 0) - AH->OF = fdopen(dup(fn), PG_BINARY_A); - else - AH->OF = fopen(filename, PG_BINARY_A); - } - else - { - if (fn >= 0) - AH->OF = fdopen(dup(fn), PG_BINARY_W); - else - AH->OF = fopen(filename, PG_BINARY_W); - } - AH->gzOut = 0; - } + AH->OF = cfopen(filename, mode, compressionMethod, compressionLevel); if (!AH->OF) { @@ -1550,33 +1527,24 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression) } } -static OutputContext +static cfp * SaveOutput(ArchiveHandle *AH) { - OutputContext sav; - - sav.OF = AH->OF; - sav.gzOut = AH->gzOut; - - return sav; + return (cfp *)AH->OF; } static void -RestoreOutput(ArchiveHandle *AH, OutputContext savedContext) +RestoreOutput(ArchiveHandle *AH, cfp *savedOutput) { int res; - errno = 0; /* in case gzclose() doesn't set it */ - if (AH->gzOut) - res = GZCLOSE(AH->OF); - else - res = fclose(AH->OF); + errno = 0; + res = cfclose(AH->OF); if (res != 0) fatal("could not close output file: %m"); - AH->gzOut = savedContext.gzOut; - AH->OF = savedContext.OF; + AH->OF = savedOutput; } @@ -1700,22 +1668,16 @@ ahwrite(const void *ptr, size_t size, size_t nmemb, ArchiveHandle *AH) bytes_written = size * nmemb; } - else if (AH->gzOut) - bytes_written = GZWRITE(ptr, size, nmemb, AH->OF); else if (AH->CustomOutPtr) bytes_written = AH->CustomOutPtr(AH, ptr, size * nmemb); - - else - { - /* - * If we're doing a restore, and it's direct to DB, and we're - * connected then send it to the DB. - */ - if (RestoringToDB(AH)) + /* + * If we're doing a restore, and it's direct to DB, and we're + * connected then send it to the DB. + */ + else if (RestoringToDB(AH)) bytes_written = ExecuteSqlCommandBuf(&AH->public, (const char *) ptr, size * nmemb); - else - bytes_written = fwrite(ptr, size, nmemb, AH->OF) * size; - } + else + bytes_written = cfwrite(ptr, size * nmemb, AH->OF); if (bytes_written != size * nmemb) WRITE_ERROR_EXIT; @@ -2200,7 +2162,9 @@ _discoverArchiveFormat(ArchiveHandle *AH) */ static ArchiveHandle * _allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + const CompressionMethod compressionMethod, + const int compressionLevel, + bool dosync, ArchiveMode mode, SetupWorkerPtrType setupWorkerPtr) { ArchiveHandle *AH; @@ -2251,14 +2215,14 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, AH->toc->prev = AH->toc; AH->mode = mode; - AH->compression = compression; + AH->compressionMethod = compressionMethod; + AH->compressionLevel = compressionLevel; AH->dosync = dosync; memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse)); /* Open stdout with no compression for AH output handle */ - AH->gzOut = 0; - AH->OF = stdout; + AH->OF = cfdopen(dup(fileno(stdout)), PG_BINARY_A, COMPRESSION_NONE, 0); /* * On Windows, we need to use binary mode to read/write non-text files, @@ -2266,7 +2230,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, * Force stdin/stdout into binary mode if that is what we are using. */ #ifdef WIN32 - if ((fmt != archNull || compression != 0) && + if ((fmt != archNull || compressionMethod != COMPRESSION_NONE) && (AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0)) { if (mode == archModeWrite) @@ -3716,7 +3680,7 @@ WriteHead(ArchiveHandle *AH) AH->WriteBytePtr(AH, AH->intSize); AH->WriteBytePtr(AH, AH->offSize); AH->WriteBytePtr(AH, AH->format); - WriteInt(AH, AH->compression); + WriteInt(AH, AH->compressionLevel); crtm = *localtime(&AH->createDate); WriteInt(AH, crtm.tm_sec); WriteInt(AH, crtm.tm_min); @@ -3790,18 +3754,21 @@ ReadHead(ArchiveHandle *AH) if (AH->version >= K_VERS_1_2) { if (AH->version < K_VERS_1_4) - AH->compression = AH->ReadBytePtr(AH); + AH->compressionLevel = AH->ReadBytePtr(AH); else - AH->compression = ReadInt(AH); + AH->compressionLevel = ReadInt(AH); } else - AH->compression = Z_DEFAULT_COMPRESSION; + AH->compressionLevel = Z_DEFAULT_COMPRESSION; + if (AH->compressionLevel != INT_MIN) #ifndef HAVE_LIBZ - if (AH->compression != 0) pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available"); +#else + AH->compressionMethod = COMPRESSION_GZIP; #endif + if (AH->version >= K_VERS_1_4) { struct tm crtm; diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h index 540d4f6a83..837e9d73f5 100644 --- a/src/bin/pg_dump/pg_backup_archiver.h +++ b/src/bin/pg_dump/pg_backup_archiver.h @@ -32,30 +32,6 @@ #define LOBBUFSIZE 16384 -#ifdef HAVE_LIBZ -#include -#define GZCLOSE(fh) gzclose(fh) -#define GZWRITE(p, s, n, fh) gzwrite(fh, p, (n) * (s)) -#define GZREAD(p, s, n, fh) gzread(fh, p, (n) * (s)) -#define GZEOF(fh) gzeof(fh) -#else -#define GZCLOSE(fh) fclose(fh) -#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s)) -#define GZREAD(p, s, n, fh) fread(p, s, n, fh) -#define GZEOF(fh) feof(fh) -/* this is just the redefinition of a libz constant */ -#define Z_DEFAULT_COMPRESSION (-1) - -typedef struct _z_stream -{ - void *next_in; - void *next_out; - size_t avail_in; - size_t avail_out; -} z_stream; -typedef z_stream *z_streamp; -#endif - /* Data block types */ #define BLK_DATA 1 #define BLK_BLOBS 3 @@ -319,8 +295,7 @@ struct _archiveHandle char *fSpec; /* Archive File Spec */ FILE *FH; /* General purpose file handle */ - void *OF; - int gzOut; /* Output file */ + void *OF; /* Output file */ struct _tocEntry *toc; /* Header of circular list of TOC entries */ int tocCount; /* Number of TOC entries */ @@ -331,14 +306,17 @@ struct _archiveHandle DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */ struct _tocEntry *currToc; /* Used when dumping data */ - int compression; /*--------- - * Compression requested on open(). - * Possible values for compression: - * -1 Z_DEFAULT_COMPRESSION - * 0 COMPRESSION_NONE - * 1-9 levels for gzip compression - *--------- - */ + CompressionMethod compressionMethod; /* Requested method for compression */ + int compressionLevel; /*--------- + * Requested level of compression for method. + * Possible values for compression: + * INT_MIN when no compression method is + * requested. + * -1 Z_DEFAULT_COMPRESSION for gzip + * compression. + * 1-9 levels for gzip compression. + *--------- + */ bool dosync; /* data requested to be synced on sight */ ArchiveMode mode; /* File mode - r or w */ void *formatData; /* Header data specific to file format */ diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c index 77d402c323..7f38ea9cd5 100644 --- a/src/bin/pg_dump/pg_backup_custom.c +++ b/src/bin/pg_dump/pg_backup_custom.c @@ -298,7 +298,9 @@ _StartData(ArchiveHandle *AH, TocEntry *te) _WriteByte(AH, BLK_DATA); /* Block type */ WriteInt(AH, te->dumpId); /* For sanity check */ - ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc); + ctx->cs = AllocateCompressor(AH->compressionMethod, + AH->compressionLevel, + _CustomWriteFunc); } /* @@ -377,7 +379,9 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) WriteInt(AH, oid); - ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc); + ctx->cs = AllocateCompressor(AH->compressionMethod, + AH->compressionLevel, + _CustomWriteFunc); } /* @@ -566,7 +570,8 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te) static void _PrintData(ArchiveHandle *AH) { - ReadDataFromArchive(AH, AH->compression, _CustomReadFunc); + ReadDataFromArchive(AH, AH->compressionMethod, AH->compressionLevel, + _CustomReadFunc); } static void diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c index 7f4e340dea..0e60b447de 100644 --- a/src/bin/pg_dump/pg_backup_directory.c +++ b/src/bin/pg_dump/pg_backup_directory.c @@ -327,7 +327,9 @@ _StartData(ArchiveHandle *AH, TocEntry *te) setFilePath(AH, fname, tctx->filename); - ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression); + ctx->dataFH = cfopen_write(fname, PG_BINARY_W, + AH->compressionMethod, + AH->compressionLevel); if (ctx->dataFH == NULL) fatal("could not open output file \"%s\": %m", fname); } @@ -581,7 +583,8 @@ _CloseArchive(ArchiveHandle *AH) ctx->pstate = ParallelBackupStart(AH); /* The TOC is always created uncompressed */ - tocFH = cfopen_write(fname, PG_BINARY_W, 0); + tocFH = cfopen_write(fname, PG_BINARY_W, + COMPRESSION_NONE, 0); if (tocFH == NULL) fatal("could not open output file \"%s\": %m", fname); ctx->dataFH = tocFH; @@ -644,7 +647,7 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te) setFilePath(AH, fname, "blobs.toc"); /* The blob TOC file is never compressed */ - ctx->blobsTocFH = cfopen_write(fname, "ab", 0); + ctx->blobsTocFH = cfopen_write(fname, "ab", COMPRESSION_NONE, 0); if (ctx->blobsTocFH == NULL) fatal("could not open output file \"%s\": %m", fname); } @@ -662,7 +665,8 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid); - ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression); + ctx->dataFH = cfopen_write(fname, PG_BINARY_W, + AH->compressionMethod, AH->compressionLevel); if (ctx->dataFH == NULL) fatal("could not open output file \"%s\": %m", fname); diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c index 2491a091b9..b25b641caa 100644 --- a/src/bin/pg_dump/pg_backup_tar.c +++ b/src/bin/pg_dump/pg_backup_tar.c @@ -35,6 +35,7 @@ #include #include "common/file_utils.h" +#include "compress_io.h" #include "fe_utils/string_utils.h" #include "pg_backup_archiver.h" #include "pg_backup_tar.h" @@ -194,7 +195,7 @@ InitArchiveFmt_Tar(ArchiveHandle *AH) * possible since gzdopen uses buffered IO which totally screws file * positioning. */ - if (AH->compression != 0) + if (AH->compressionMethod != COMPRESSION_NONE) fatal("compression is not supported by tar archive format"); } else @@ -328,7 +329,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) } } - if (AH->compression == 0) + if (AH->compressionMethod == COMPRESSION_NONE) tm->nFH = ctx->tarFH; else fatal("compression is not supported by tar archive format"); @@ -383,7 +384,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) umask(old_umask); - if (AH->compression == 0) + if (AH->compressionMethod == COMPRESSION_NONE) tm->nFH = tm->tmpFH; else fatal("compression is not supported by tar archive format"); @@ -401,7 +402,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) static void tarClose(ArchiveHandle *AH, TAR_MEMBER *th) { - if (AH->compression != 0) + if (AH->compressionMethod != COMPRESSION_NONE) fatal("compression is not supported by tar archive format"); if (th->mode == 'w') @@ -801,7 +802,6 @@ _CloseArchive(ArchiveHandle *AH) memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions)); ropt->filename = NULL; ropt->dropSchema = 1; - ropt->compression = 0; ropt->superuser = NULL; ropt->suppressDumpWarnings = true; @@ -889,7 +889,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) if (oid == 0) fatal("invalid OID for large object (%u)", oid); - if (AH->compression != 0) + if (AH->compressionMethod != COMPRESSION_NONE) fatal("compression is not supported by tar archive format"); sprintf(fname, "blob_%u.dat", oid); diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 535b160165..97ac17ebff 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -55,6 +55,7 @@ #include "catalog/pg_trigger_d.h" #include "catalog/pg_type_d.h" #include "common/connect.h" +#include "compress_io.h" #include "dumputils.h" #include "fe_utils/option_utils.h" #include "fe_utils/string_utils.h" @@ -163,6 +164,9 @@ static void setup_connection(Archive *AH, const char *dumpencoding, const char *dumpsnapshot, char *use_role); static ArchiveFormat parseArchiveFormat(const char *format, ArchiveMode *mode); +static bool parse_compression_option(const char *opt, + CompressionMethod *compressionMethod, + int *compressLevel); static void expand_schema_name_patterns(Archive *fout, SimpleStringList *patterns, SimpleOidList *oids, @@ -336,8 +340,9 @@ main(int argc, char **argv) const char *dumpsnapshot = NULL; char *use_role = NULL; int numWorkers = 1; - int compressLevel = -1; int plainText = 0; + int compressLevel = INT_MIN; + CompressionMethod compressionMethod = COMPRESSION_INVALID; ArchiveFormat archiveFormat = archUnknown; ArchiveMode archiveMode; @@ -557,9 +562,9 @@ main(int argc, char **argv) dopt.aclsSkip = true; break; - case 'Z': /* Compression Level */ - if (!option_parse_int(optarg, "-Z/--compress", 0, 9, - &compressLevel)) + case 'Z': /* Compression */ + if (!parse_compression_option(optarg, &compressionMethod, + &compressLevel)) exit_nicely(1); break; @@ -689,23 +694,21 @@ main(int argc, char **argv) if (archiveFormat == archNull) plainText = 1; - /* Custom and directory formats are compressed by default, others not */ - if (compressLevel == -1) + /* Set default compressionMethod unless one already set by the user */ + if (compressionMethod == COMPRESSION_INVALID) { + compressionMethod = COMPRESSION_NONE; + #ifdef HAVE_LIBZ + /* Custom and directory formats are compressed by default (zlib) */ if (archiveFormat == archCustom || archiveFormat == archDirectory) + { + compressionMethod = COMPRESSION_GZIP; compressLevel = Z_DEFAULT_COMPRESSION; - else + } #endif - compressLevel = 0; } -#ifndef HAVE_LIBZ - if (compressLevel != 0) - pg_log_warning("requested compression not available in this installation -- archive will be uncompressed"); - compressLevel = 0; -#endif - /* * If emitting an archive format, we always want to emit a DATABASE item, * in case --create is specified at pg_restore time. @@ -718,8 +721,9 @@ main(int argc, char **argv) fatal("parallel backup only supported by the directory format"); /* Open the output file */ - fout = CreateArchive(filename, archiveFormat, compressLevel, dosync, - archiveMode, setupDumpWorker); + fout = CreateArchive(filename, archiveFormat, + compressionMethod, compressLevel, + dosync, archiveMode, setupDumpWorker); /* Make dump options accessible right away */ SetArchiveOptions(fout, &dopt, NULL); @@ -950,10 +954,8 @@ main(int argc, char **argv) ropt->sequence_data = dopt.sequence_data; ropt->binary_upgrade = dopt.binary_upgrade; - if (compressLevel == -1) - ropt->compression = 0; - else - ropt->compression = compressLevel; + ropt->compressionLevel = compressLevel; + ropt->compressionMethod = compressionMethod; ropt->suppressDumpWarnings = true; /* We've already shown them */ @@ -1000,7 +1002,8 @@ help(const char *progname) printf(_(" -j, --jobs=NUM use this many parallel jobs to dump\n")); printf(_(" -v, --verbose verbose mode\n")); printf(_(" -V, --version output version information, then exit\n")); - printf(_(" -Z, --compress=0-9 compression level for compressed formats\n")); + printf(_(" -Z, --compress=[gzip,none][:LEVEL] or [LEVEL]\n" + " compress output with given method or level\n")); printf(_(" --lock-wait-timeout=TIMEOUT fail after waiting TIMEOUT for a table lock\n")); printf(_(" --no-sync do not wait for changes to be written safely to disk\n")); printf(_(" -?, --help show this help, then exit\n")); @@ -1260,6 +1263,116 @@ get_synchronized_snapshot(Archive *fout) return result; } +static bool +parse_compression_method(const char *method, + CompressionMethod *compressionMethod) +{ + bool res = true; + + if (pg_strcasecmp(method, "gzip") == 0) + *compressionMethod = COMPRESSION_GZIP; + else if (pg_strcasecmp(method, "none") == 0) + *compressionMethod = COMPRESSION_NONE; + else + { + pg_log_error("invalid compression method \"%s\" (gzip, none)", method); + res = false; + } + + return res; +} + +/* + * Interprets a compression option of the format 'method[:LEVEL]' of legacy just + * '[LEVEL]'. In the later format, gzip is implied. The parsed method and level + * are returned in *compressionMethod and *compressionLevel. In case of error, + * the function returns false and then the values of *compression{Method,Level} + * are not to be trusted. + */ +static bool +parse_compression_option(const char *opt, CompressionMethod *compressionMethod, + int *compressLevel) +{ + char *method; + const char *sep; + int methodlen; + bool supports_compression = true; + bool res = true; + + /* find the separator if exists */ + sep = strchr(opt, ':'); + + /* + * If there is no separator, then it is either a legacy format, or only the + * method has been passed. + */ + if (!sep) + { + if (strspn(opt, "-0123456789") == strlen(opt)) + { + res = option_parse_int(opt, "-Z/--compress", 0, 9, compressLevel); + *compressionMethod = (*compressLevel > 0) ? COMPRESSION_GZIP : + COMPRESSION_NONE; + } + else + res = parse_compression_method(opt, compressionMethod); + } + else + { + /* otherwise, it should be method:LEVEL */ + methodlen = sep - opt + 1; + method = pg_malloc0(methodlen); + snprintf(method, methodlen, "%.*s", methodlen - 1, opt); + + res = parse_compression_method(method, compressionMethod); + if (res) + { + sep++; + if (*sep == '\0') + { + pg_log_error("no level defined for compression \"%s\"", method); + pg_free(method); + res = false; + } + else + { + res = option_parse_int(sep, "-Z/--compress [LEVEL]", 1, 9, + compressLevel); + } + } + } + + /* if there is an error, there is no need to check further */ + if (!res) + return res; + + /* one can set level when method is gzip */ + if (*compressionMethod != COMPRESSION_GZIP && *compressLevel != INT_MIN) + { + pg_log_error("can only specify -Z/--compress [LEVEL] when method is gzip"); + return false; + } + + /* verify that the requested compression is supported */ +#ifndef HAVE_LIBZ + if (*compressionMethod == COMPRESSION_GZIP) + supports_compression = false; +#endif +#ifndef HAVE_LIBLZ4 + if (*compressionMethod == COMPRESSION_LZ4) + supports_compression = false; +#endif + + if (!supports_compression) + { + pg_log_warning("requested compression not available in this installation -- archive will be uncompressed"); + *compressionMethod = COMPRESSION_NONE; + *compressLevel = INT_MIN; + } + + return true; +} + static ArchiveFormat parseArchiveFormat(const char *format, ArchiveMode *mode) { diff --git a/src/bin/pg_dump/t/001_basic.pl b/src/bin/pg_dump/t/001_basic.pl index 65e6c01fed..d7a52ac1a8 100644 --- a/src/bin/pg_dump/t/001_basic.pl +++ b/src/bin/pg_dump/t/001_basic.pl @@ -120,6 +120,16 @@ command_fails_like( qr/\Qpg_restore: error: cannot specify both --single-transaction and multiple jobs\E/, 'pg_restore: cannot specify both --single-transaction and multiple jobs'); +command_fails_like( + [ 'pg_dump', '--compress', 'garbage' ], + qr/\Qpg_dump: error: invalid compression method "garbage" (gzip, none)\E/, + 'pg_dump: invalid --compress'); + +command_fails_like( + [ 'pg_dump', '--compress', 'none:1' ], + qr/\Qpg_dump: error: can only specify -Z\/--compress [LEVEL] when method is gzip\E/, + 'pg_dump: can only specify -Z/--compress [LEVEL] when method is gzip'); + command_fails_like( [ 'pg_dump', '-Z', '-1' ], qr/\Qpg_dump: error: -Z\/--compress must be in range 0..9\E/, @@ -128,7 +138,7 @@ command_fails_like( if (check_pg_config("#define HAVE_LIBZ 1")) { command_fails_like( - [ 'pg_dump', '--compress', '1', '--format', 'tar' ], + [ 'pg_dump', '--compress', 'gzip:1', '--format', 'tar' ], qr/\Qpg_dump: error: compression is not supported by tar archive format\E/, 'pg_dump: compression is not supported by tar archive format'); } @@ -136,7 +146,7 @@ else { # --jobs > 1 forces an error with tar format. command_fails_like( - [ 'pg_dump', '--compress', '1', '--format', 'tar', '-j3' ], + [ 'pg_dump', '--compress', 'gzip:1', '--format', 'tar', '-j3' ], qr/\Qpg_dump: warning: requested compression not available in this installation -- archive will be uncompressed\E/, 'pg_dump: warning: compression not available in this installation'); } diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl index 91c3b978c4..3a3dad2cc7 100644 --- a/src/bin/pg_dump/t/002_pg_dump.pl +++ b/src/bin/pg_dump/t/002_pg_dump.pl @@ -83,7 +83,7 @@ my %pgdump_runs = ( test_key => 'compression', dump_cmd => [ 'pg_dump', - '--format=directory', '--compress=1', + '--format=directory', '--compress=gzip:1', "--file=$tempdir/compression_gzip_directory_format", 'postgres', ], @@ -104,7 +104,7 @@ my %pgdump_runs = ( test_key => 'compression', dump_cmd => [ 'pg_dump', '--jobs=2', - '--format=directory', '--compress=6', + '--format=directory', '--compress=gzip:6', "--file=$tempdir/compression_gzip_directory_format_parallel", 'postgres', ], @@ -137,6 +137,25 @@ my %pgdump_runs = ( "$tempdir/compression_gzip_plain_format.sql.gz", ], }, + compression_none_dir_format => { + test_key => 'compression', + dump_cmd => [ + 'pg_dump', '-Fd', + '--compress=none', + "--file=$tempdir/compression_none_dir_format", + 'postgres', + ], + glob_match => { + no_match => "$tempdir/compression_none_dir_format/*.dat.gz", + match => "$tempdir/compression_none_dir_format/*.dat", + match_count => 2, # toc.dat and more + }, + restore_cmd => [ + 'pg_restore', '-Fd', + "--file=$tempdir/compression_none_dir_format.sql", + "$tempdir/compression_none_dir_format", + ], + }, clean => { dump_cmd => [ 'pg_dump', @@ -236,7 +255,7 @@ my %pgdump_runs = ( defaults_dir_format => { test_key => 'defaults', dump_cmd => [ - 'pg_dump', '-Fd', + 'pg_dump', '-Fd', "--file=$tempdir/defaults_dir_format", 'postgres', ], restore_cmd => [ @@ -4048,6 +4067,26 @@ foreach my $run (sort keys %pgdump_runs) command_ok( \@{ $pgdump_runs{$run}->{compress_cmd} }, "$run: compression commands"); + + if (defined($pgdump_runs{$run}->{glob_match})) + { + my $match = $pgdump_runs{$run}->{glob_match}->{match}; + my $match_count = defined($pgdump_runs{$run}->{glob_match}->{match_count}) ? + $pgdump_runs{$run}->{glob_match}->{match_count} : 1; + my @glob_matched = glob $match; + + cmp_ok(scalar(@glob_matched), '>=', $match_count, + "Expected at least $match_count file(s) matching $match"); + + if (defined($pgdump_runs{$run}->{glob_match}->{no_match})) + { + my $no_match = $pgdump_runs{$run}->{glob_match}->{no_match}; + my @glob_matched = glob $no_match; + + cmp_ok(scalar(@glob_matched), '==', 0, + "Expected no file(s) matching $no_match"); + } + } } if ($pgdump_runs{$run}->{restore_cmd}) diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 72fafb795b..5ac8c156a9 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -412,7 +412,7 @@ CompiledExprState CompositeIOData CompositeTypeStmt CompoundAffixFlag -CompressionAlgorithm +CompressionMethod CompressorState ComputeXidHorizonsResult ConditionVariable -- 2.32.0