From b2beb69c23ea8346a684e8c7ae5a5b60bade2066 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Thu, 6 Jan 2022 13:46:56 +0900 Subject: [PATCH v3 1/2] Refactor walmethods.c's tar method to use compression method Some TAP tests are added for pg_basebackup --compress, while on it. --- src/bin/pg_basebackup/pg_basebackup.c | 3 +- src/bin/pg_basebackup/t/010_pg_basebackup.pl | 36 ++++++++++++++- src/bin/pg_basebackup/walmethods.c | 47 +++++++++++++------- 3 files changed, 67 insertions(+), 19 deletions(-) diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c index 1739ac6382..17ff0132d9 100644 --- a/src/bin/pg_basebackup/pg_basebackup.c +++ b/src/bin/pg_basebackup/pg_basebackup.c @@ -524,7 +524,8 @@ LogStreamerMain(logstreamer_param *param) stream.do_sync); else stream.walmethod = CreateWalTarMethod(param->xlog, - COMPRESSION_NONE, /* ignored */ + (compresslevel > 0) ? + COMPRESSION_GZIP : COMPRESSION_NONE, compresslevel, stream.do_sync); diff --git a/src/bin/pg_basebackup/t/010_pg_basebackup.pl b/src/bin/pg_basebackup/t/010_pg_basebackup.pl index e56825382c..192781e0a2 100644 --- a/src/bin/pg_basebackup/t/010_pg_basebackup.pl +++ b/src/bin/pg_basebackup/t/010_pg_basebackup.pl @@ -10,7 +10,7 @@ use File::Path qw(rmtree); use Fcntl qw(:seek); use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; -use Test::More tests => 110; +use Test::More tests => 113; program_help_ok('pg_basebackup'); program_version_ok('pg_basebackup'); @@ -624,3 +624,37 @@ rmtree("$tempdir/backup_corrupt4"); $node->safe_psql('postgres', "DROP TABLE corrupt1;"); $node->safe_psql('postgres', "DROP TABLE corrupt2;"); + +note "Testing pg_basebackup with compression methods"; + +# Check ZLIB compression if available. +SKIP: +{ + skip "postgres was not built with ZLIB support", 3 + if (!check_pg_config("#define HAVE_LIBZ 1")); + + $node->command_ok( + [ + 'pg_basebackup', '-D', + "$tempdir/backup_gzip", '--compress', '1', + '--no-sync', '--format', 't' + ], + 'pg_basebackup with --compress'); + + # Verify that the stored files are generated with their expected + # names. + my @zlib_files = glob "$tempdir/backup_gzip/*.tar.gz"; + is(scalar(@zlib_files), 2, + "two files created with gzip (base.tar.gz and pg_wal.tar.gz)"); + + # Check the integrity of the files generated. + my $gzip = $ENV{GZIP_PROGRAM}; + skip "program gzip is not found in your system", 1 + if ( !defined $gzip + || $gzip eq '' + || system_log($gzip, '--version') != 0); + + my $gzip_is_valid = system_log($gzip, '--test', @zlib_files); + is($gzip_is_valid, 0, "gzip verified the integrity of compressed data"); + rmtree("$tempdir/backup_gzip"); +} diff --git a/src/bin/pg_basebackup/walmethods.c b/src/bin/pg_basebackup/walmethods.c index affdc5055f..127bb5e013 100644 --- a/src/bin/pg_basebackup/walmethods.c +++ b/src/bin/pg_basebackup/walmethods.c @@ -749,7 +749,7 @@ tar_write(Walfile f, const void *buf, size_t count) tar_clear_error(); /* Tarfile will always be positioned at the end */ - if (!tar_data->compression_level) + if (tar_data->compression_method == COMPRESSION_NONE) { errno = 0; r = write(tar_data->fd, buf, count); @@ -833,7 +833,7 @@ tar_open_for_write(const char *pathname, const char *temp_suffix, size_t pad_to_ } #ifdef HAVE_LIBZ - if (tar_data->compression_level) + if (tar_data->compression_method == COMPRESSION_GZIP) { tar_data->zp = (z_streamp) pg_malloc(sizeof(z_stream)); tar_data->zp->zalloc = Z_NULL; @@ -884,7 +884,7 @@ tar_open_for_write(const char *pathname, const char *temp_suffix, size_t pad_to_ pg_free(tmppath); #ifdef HAVE_LIBZ - if (tar_data->compression_level) + if (tar_data->compression_method == COMPRESSION_GZIP) { /* Flush existing data */ if (!tar_write_compressed_data(NULL, 0, true)) @@ -909,7 +909,7 @@ tar_open_for_write(const char *pathname, const char *temp_suffix, size_t pad_to_ } tar_data->currentfile->currpos = 0; - if (!tar_data->compression_level) + if (tar_data->compression_method == COMPRESSION_NONE) { errno = 0; if (write(tar_data->fd, tar_data->currentfile->header, @@ -923,7 +923,7 @@ tar_open_for_write(const char *pathname, const char *temp_suffix, size_t pad_to_ } } #ifdef HAVE_LIBZ - else + else if (tar_data->compression_method == COMPRESSION_GZIP) { /* Write header through the zlib APIs but with no compression */ if (!tar_write_compressed_data(tar_data->currentfile->header, @@ -938,6 +938,10 @@ tar_open_for_write(const char *pathname, const char *temp_suffix, size_t pad_to_ } } #endif + else + { + Assert(false); /* not reachable */ + } tar_data->currentfile->pathname = pg_strdup(pathname); @@ -948,7 +952,7 @@ tar_open_for_write(const char *pathname, const char *temp_suffix, size_t pad_to_ if (pad_to_size) { tar_data->currentfile->pad_to_size = pad_to_size; - if (!tar_data->compression_level) + if (tar_data->compression_method == COMPRESSION_NONE) { /* Uncompressed, so pad now */ if (!tar_write_padding_data(tar_data->currentfile, pad_to_size)) @@ -1009,7 +1013,7 @@ tar_sync(Walfile f) * Always sync the whole tarfile, because that's all we can do. This makes * no sense on compressed files, so just ignore those. */ - if (tar_data->compression_level) + if (tar_data->compression_method != COMPRESSION_NONE) return 0; r = fsync(tar_data->fd); @@ -1030,7 +1034,7 @@ tar_close(Walfile f, WalCloseMethod method) if (method == CLOSE_UNLINK) { - if (tar_data->compression_level) + if (tar_data->compression_method != COMPRESSION_NONE) { tar_set_error("unlink not supported with compression"); return -1; @@ -1061,7 +1065,7 @@ tar_close(Walfile f, WalCloseMethod method) */ if (tf->pad_to_size) { - if (tar_data->compression_level) + if (tar_data->compression_method == COMPRESSION_GZIP) { /* * A compressed tarfile is padded on close since we cannot know @@ -1102,7 +1106,7 @@ tar_close(Walfile f, WalCloseMethod method) #ifdef HAVE_LIBZ - if (tar_data->compression_level) + if (tar_data->compression_method == COMPRESSION_GZIP) { /* Flush the current buffer */ if (!tar_write_compressed_data(NULL, 0, true)) @@ -1131,7 +1135,7 @@ tar_close(Walfile f, WalCloseMethod method) tar_data->lasterrno = errno; return -1; } - if (!tar_data->compression_level) + if (tar_data->compression_method == COMPRESSION_NONE) { errno = 0; if (write(tar_data->fd, tf->header, TAR_BLOCK_SIZE) != TAR_BLOCK_SIZE) @@ -1142,7 +1146,7 @@ tar_close(Walfile f, WalCloseMethod method) } } #ifdef HAVE_LIBZ - else + else if (tar_data->compression_method == COMPRESSION_GZIP) { /* Turn off compression */ if (deflateParams(tar_data->zp, 0, 0) != Z_OK) @@ -1164,6 +1168,10 @@ tar_close(Walfile f, WalCloseMethod method) } } #endif + else + { + Assert(false); /* not reachable */ + } /* Move file pointer back down to end, so we can write the next file */ if (lseek(tar_data->fd, 0, SEEK_END) < 0) @@ -1212,7 +1220,7 @@ tar_finish(void) /* A tarfile always ends with two empty blocks */ MemSet(zerobuf, 0, sizeof(zerobuf)); - if (!tar_data->compression_level) + if (tar_data->compression_method == COMPRESSION_NONE) { errno = 0; if (write(tar_data->fd, zerobuf, sizeof(zerobuf)) != sizeof(zerobuf)) @@ -1223,7 +1231,7 @@ tar_finish(void) } } #ifdef HAVE_LIBZ - else + else if (tar_data->compression_method == COMPRESSION_GZIP) { if (!tar_write_compressed_data(zerobuf, sizeof(zerobuf), false)) return false; @@ -1268,6 +1276,10 @@ tar_finish(void) } } #endif + else + { + Assert(false); /* not reachable */ + } /* sync the empty blocks as well, since they're after the last file */ if (tar_data->sync) @@ -1312,7 +1324,8 @@ CreateWalTarMethod(const char *tarbase, int compression_level, bool sync) { WalWriteMethod *method; - const char *suffix = (compression_level != 0) ? ".tar.gz" : ".tar"; + const char *suffix = (compression_method == COMPRESSION_GZIP) ? + ".tar.gz" : ".tar"; method = pg_malloc0(sizeof(WalWriteMethod)); method->open_for_write = tar_open_for_write; @@ -1335,7 +1348,7 @@ CreateWalTarMethod(const char *tarbase, tar_data->compression_level = compression_level; tar_data->sync = sync; #ifdef HAVE_LIBZ - if (compression_level) + if (compression_method == COMPRESSION_GZIP) tar_data->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1); #endif @@ -1347,7 +1360,7 @@ FreeWalTarMethod(void) { pg_free(tar_data->tarfilename); #ifdef HAVE_LIBZ - if (tar_data->compression_level) + if (tar_data->compression_method == COMPRESSION_GZIP) pg_free(tar_data->zlibOut); #endif pg_free(tar_data); -- 2.34.1