From 093f9104e1e50b777e05f24665c7c1125d97ace3 Mon Sep 17 00:00:00 2001 From: Georgios Kokolatos Date: Fri, 1 Apr 2022 13:15:14 +0000 Subject: [PATCH v5 1/3] Extend compression coverage for pg_dump, pg_restore --- src/bin/pg_dump/Makefile | 2 + src/bin/pg_dump/t/002_pg_dump.pl | 110 +++++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+) diff --git a/src/bin/pg_dump/Makefile b/src/bin/pg_dump/Makefile index 302f7e02d6..2f524b09bf 100644 --- a/src/bin/pg_dump/Makefile +++ b/src/bin/pg_dump/Makefile @@ -16,6 +16,8 @@ subdir = src/bin/pg_dump top_builddir = ../../.. include $(top_builddir)/src/Makefile.global +export GZIP_PROGRAM=$(GZIP) + override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS) LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport) diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl index af5d6fa5a3..134cc0618b 100644 --- a/src/bin/pg_dump/t/002_pg_dump.pl +++ b/src/bin/pg_dump/t/002_pg_dump.pl @@ -26,6 +26,13 @@ my $tempdir = PostgreSQL::Test::Utils::tempdir; # specified and is pulled from $PGPORT, which is set by the # PostgreSQL::Test::Cluster system. # +# compress_cmd is the utility command for (de)compression, if any. +# Note that this should generally be used on pg_dump's output +# either to generate a text file to run the through the tests, or +# to test pg_restore's ability to parse manually compressed files +# that otherwise pg_dump does not compress on it's own +# (e.g. *.toc). +# # restore_cmd is the pg_restore command to run, if any. Note # that this should generally be used when the pg_dump goes to # a non-text file and that the restore can then be used to @@ -54,6 +61,88 @@ my %pgdump_runs = ( "$tempdir/binary_upgrade.dump", ], }, + + # Do not use --no-sync to give test coverage for data sync. + compression_gzip_custom_format => { + test_key => 'compression', + dump_cmd => [ + 'pg_dump', + '--format=custom', '--compress=1', + "--file=$tempdir/compression_gzip_custom_format.dump", + 'postgres', + ], + restore_cmd => [ + 'pg_restore', + "--file=$tempdir/compression_gzip_custom_format.sql", + "$tempdir/compression_gzip_custom_format.dump", + ], + }, + + # Do not use --no-sync to give test coverage for data sync. + compression_gzip_directory_format => { + test_key => 'compression', + dump_cmd => [ + 'pg_dump', + '--format=directory', '--compress=1', + "--file=$tempdir/compression_gzip_directory_format", + 'postgres', + ], + # Give coverage for manually compressed blob.toc files during restore. + compress_cmd => { + program => $ENV{'GZIP_PROGRAM'}, + args => [ + '-f', + "$tempdir/compression_gzip_directory_format/blobs.toc", + ], + }, + restore_cmd => [ + 'pg_restore', + "--file=$tempdir/compression_gzip_directory_format.sql", + "$tempdir/compression_gzip_directory_format", + ], + }, + + compression_gzip_directory_format_parallel => { + test_key => 'compression', + dump_cmd => [ + 'pg_dump', '--jobs=2', + '--format=directory', '--compress=6', + "--file=$tempdir/compression_gzip_directory_format_parallel", + 'postgres', + ], + # Give coverage for manually compressed blob.toc files during restore. + compress_cmd => { + program => $ENV{'GZIP_PROGRAM'}, + args => [ + '-f', + "$tempdir/compression_gzip_directory_format_parallel/blobs.toc", + ] + }, + restore_cmd => [ + 'pg_restore', + '--jobs=3', + "--file=$tempdir/compression_gzip_directory_format_parallel.sql", + "$tempdir/compression_gzip_directory_format_parallel", + ], + }, + + # Check that the output is valid gzip + compression_gzip_plain_format => { + test_key => 'compression', + dump_cmd => [ + 'pg_dump', '--format=plain', '-Z1', + "--file=$tempdir/compression_gzip_plain_format.sql.gz", + 'postgres', + ], + # Decompress the generated file to run through the tests + compress_cmd => { + program => $ENV{'GZIP_PROGRAM'}, + args => [ + '-d', + "$tempdir/compression_gzip_plain_format.sql.gz", + ], + }, + }, clean => { dump_cmd => [ 'pg_dump', @@ -424,6 +513,7 @@ my %full_runs = ( binary_upgrade => 1, clean => 1, clean_if_exists => 1, + compression => 1, createdb => 1, defaults => 1, exclude_dump_test_schema => 1, @@ -3098,6 +3188,7 @@ my %tests = ( binary_upgrade => 1, clean => 1, clean_if_exists => 1, + compression => 1, createdb => 1, defaults => 1, exclude_test_table => 1, @@ -3171,6 +3262,7 @@ my %tests = ( binary_upgrade => 1, clean => 1, clean_if_exists => 1, + compression => 1, createdb => 1, defaults => 1, exclude_dump_test_schema => 1, @@ -3941,6 +4033,9 @@ command_fails_like( ######################################### # Run all runs +my $supports_gzip_compression = check_pg_config("#define HAVE_LIBZ 1"); +my $gzip_program_exists = (system_log("$ENV{GZIP_PROGRAM}", '-h', + '>', '/dev/null') == 0); foreach my $run (sort keys %pgdump_runs) { @@ -3950,6 +4045,21 @@ foreach my $run (sort keys %pgdump_runs) $node->command_ok(\@{ $pgdump_runs{$run}->{dump_cmd} }, "$run: pg_dump runs"); + if ($pgdump_runs{$run}->{compress_cmd}) + { + my ($compress_cmd) = $pgdump_runs{$run}->{compress_cmd}; + + # Skip compression_cmd tests when compression is not supported, + # as the result is uncompressed or the utility program does not + # exist + next if !$supports_gzip_compression; + next if $compress_cmd->{program} eq "$ENV{GZIP_PROGRAM}" && + !$gzip_program_exists; + + my @full_cmd = ($compress_cmd->{program}, @{ $compress_cmd->{args} }); + command_ok(\@full_cmd, "$run: compression commands"); + } + if ($pgdump_runs{$run}->{restore_cmd}) { $node->command_ok(\@{ $pgdump_runs{$run}->{restore_cmd} }, -- 2.32.0