From b705f77862c9e41a149ce078df638032903bce3d Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Tue, 5 Apr 2022 10:30:06 +0900 Subject: [PATCH v6] Extend compression coverage for pg_dump, pg_restore --- src/bin/pg_dump/Makefile | 2 + src/bin/pg_dump/t/002_pg_dump.pl | 93 +++++++++++++++++++++++++++++++- 2 files changed, 93 insertions(+), 2 deletions(-) diff --git a/src/bin/pg_dump/Makefile b/src/bin/pg_dump/Makefile index 302f7e02d6..2f524b09bf 100644 --- a/src/bin/pg_dump/Makefile +++ b/src/bin/pg_dump/Makefile @@ -16,6 +16,8 @@ subdir = src/bin/pg_dump top_builddir = ../../.. include $(top_builddir)/src/Makefile.global +export GZIP_PROGRAM=$(GZIP) + override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS) LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport) diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl index af5d6fa5a3..dda3649373 100644 --- a/src/bin/pg_dump/t/002_pg_dump.pl +++ b/src/bin/pg_dump/t/002_pg_dump.pl @@ -20,12 +20,22 @@ my $tempdir = PostgreSQL::Test::Utils::tempdir; # test_key indicates that a given run should simply use the same # set of like/unlike tests as another run, and which run that is. # +# compile_option indicates if the commands run depend on a compilation +# option, if any. This controls if tests are skipped if a dependency +# is not satisfied. +# # dump_cmd is the pg_dump command to run, which is an array of # the full command and arguments to run. Note that this is run # using $node->command_ok(), so the port does not need to be # specified and is pulled from $PGPORT, which is set by the # PostgreSQL::Test::Cluster system. # +# compress_cmd is the utility command for (de)compression, if any. +# Note that this should generally be used on pg_dump's output +# either to generate a text file to run the through the tests, or +# to test pg_restore's ability to parse manually compressed files +# that otherwise pg_dump does not compress on its own (e.g. *.toc). +# # restore_cmd is the pg_restore command to run, if any. Note # that this should generally be used when the pg_dump goes to # a non-text file and that the restore can then be used to @@ -54,6 +64,58 @@ my %pgdump_runs = ( "$tempdir/binary_upgrade.dump", ], }, + + # Do not use --no-sync to give test coverage for data sync. + compression_gzip_custom => { + test_key => 'compression', + compile_option => 'gzip', + dump_cmd => [ + 'pg_dump', '--format=custom', + '--compress=1', "--file=$tempdir/compression_gzip_custom.dump", + 'postgres', + ], + restore_cmd => [ + 'pg_restore', + "--file=$tempdir/compression_gzip_custom.sql", + "$tempdir/compression_gzip_custom.dump", + ], + }, + + # Do not use --no-sync to give test coverage for data sync. + compression_gzip_dir => { + test_key => 'compression', + compile_option => 'gzip', + dump_cmd => [ + 'pg_dump', '--jobs=2', + '--format=directory', '--compress=1', + "--file=$tempdir/compression_gzip_dir", 'postgres', + ], + # Give coverage for manually compressed blob.toc files during + # restore. + compress_cmd => { + program => $ENV{'GZIP_PROGRAM'}, + args => [ '-f', "$tempdir/compression_gzip_dir/blobs.toc", ], + }, + restore_cmd => [ + 'pg_restore', '--jobs=2', + "--file=$tempdir/compression_gzip_dir.sql", + "$tempdir/compression_gzip_dir", + ], + }, + + compression_gzip_plain => { + test_key => 'compression', + compile_option => 'gzip', + dump_cmd => [ + 'pg_dump', '--format=plain', '-Z1', + "--file=$tempdir/compression_gzip_plain.sql.gz", 'postgres', + ], + # Decompress the generated file to run through the tests. + compress_cmd => { + program => $ENV{'GZIP_PROGRAM'}, + args => [ '-d', "$tempdir/compression_gzip_plain.sql.gz", ], + }, + }, clean => { dump_cmd => [ 'pg_dump', @@ -424,6 +486,7 @@ my %full_runs = ( binary_upgrade => 1, clean => 1, clean_if_exists => 1, + compression => 1, createdb => 1, defaults => 1, exclude_dump_test_schema => 1, @@ -3098,6 +3161,7 @@ my %tests = ( binary_upgrade => 1, clean => 1, clean_if_exists => 1, + compression => 1, createdb => 1, defaults => 1, exclude_test_table => 1, @@ -3171,6 +3235,7 @@ my %tests = ( binary_upgrade => 1, clean => 1, clean_if_exists => 1, + compression => 1, createdb => 1, defaults => 1, exclude_dump_test_schema => 1, @@ -3833,8 +3898,9 @@ if ($collation_check_stderr !~ /ERROR: /) $collation_support = 1; } -# Determine whether build supports LZ4. -my $supports_lz4 = check_pg_config("#define USE_LZ4 1"); +# Determine whether build supports LZ4 and gzip. +my $supports_lz4 = check_pg_config("#define USE_LZ4 1"); +my $supports_gzip = check_pg_config("#define HAVE_LIBZ 1"); # Create additional databases for mutations of schema public $node->psql('postgres', 'create database regress_pg_dump_test;'); @@ -3947,9 +4013,32 @@ foreach my $run (sort keys %pgdump_runs) my $test_key = $run; my $run_db = 'postgres'; + # Skip command-level tests for gzip if there is no support for it. + if ( defined($pgdump_runs{$run}->{compile_option}) + && $pgdump_runs{$run}->{compile_option} eq 'gzip' + && !$supports_gzip) + { + note "$run: skipped due to no gzip support"; + next; + } + $node->command_ok(\@{ $pgdump_runs{$run}->{dump_cmd} }, "$run: pg_dump runs"); + if ($pgdump_runs{$run}->{compress_cmd}) + { + my ($compress_cmd) = $pgdump_runs{$run}->{compress_cmd}; + my $compress_program = $compress_cmd->{program}; + + # Skip the rest of the test if the compression program is + # not defined. + next if (!defined($compress_program) || $compress_program eq ''); + + my @full_compress_cmd = + ($compress_cmd->{program}, @{ $compress_cmd->{args} }); + command_ok(\@full_compress_cmd, "$run: compression commands"); + } + if ($pgdump_runs{$run}->{restore_cmd}) { $node->command_ok(\@{ $pgdump_runs{$run}->{restore_cmd} }, -- 2.35.1