diff --git a/src/bin/pg_upgrade/.gitignore b/src/bin/pg_upgrade/.gitignore index 2d3bfeaa50..05200a09f1 100644 --- a/src/bin/pg_upgrade/.gitignore +++ b/src/bin/pg_upgrade/.gitignore @@ -1,9 +1,4 @@ /pg_upgrade # Generated by test suite -/pg_upgrade_internal.log -/delete_old_cluster.sh -/delete_old_cluster.bat -/reindex_hash.sql -/loadable_libraries.txt /log/ /tmp_check/ diff --git a/src/bin/pg_upgrade/Makefile b/src/bin/pg_upgrade/Makefile index 44d06be5a6..105199f182 100644 --- a/src/bin/pg_upgrade/Makefile +++ b/src/bin/pg_upgrade/Makefile @@ -28,6 +28,10 @@ OBJS = \ override CPPFLAGS := -DDLSUFFIX=\"$(DLSUFFIX)\" -I$(srcdir) -I$(libpq_srcdir) $(CPPFLAGS) LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport) +# required for 002_pg_upgrade.pl +REGRESS_SHLIB=$(abs_top_builddir)/src/test/regress/regress$(DLSUFFIX) +export REGRESS_SHLIB + all: pg_upgrade pg_upgrade: $(OBJS) | submake-libpq submake-libpgport submake-libpgfeutils @@ -44,22 +48,10 @@ uninstall: clean distclean maintainer-clean: rm -f pg_upgrade$(X) $(OBJS) - rm -rf delete_old_cluster.sh log/ tmp_check/ \ - loadable_libraries.txt reindex_hash.sql \ - pg_upgrade_dump_globals.sql \ - pg_upgrade_dump_*.custom pg_upgrade_*.log - -# When $(MAKE) is present, make automatically infers that this is a -# recursive make. which is not actually what we want here, as that -# e.g. prevents output synchronization from working (as make thinks -# that the subsidiary make knows how to deal with that itself, but -# we're invoking a shell script that doesn't know). Referencing -# $(MAKE) indirectly avoids that behaviour. -# See https://www.gnu.org/software/make/manual/html_node/MAKE-Variable.html#MAKE-Variable -NOTSUBMAKEMAKE=$(MAKE) + rm -rf log/ tmp_check/ -check: test.sh all temp-install - MAKE=$(NOTSUBMAKEMAKE) $(with_temp_install) bindir=$(abs_top_builddir)/tmp_install/$(bindir) EXTRA_REGRESS_OPTS="$(EXTRA_REGRESS_OPTS)" $(SHELL) $< +check: + $(prove_check) -# installcheck is not supported because there's no meaningful way to test -# pg_upgrade against a single already-running server +installcheck: + $(prove_installcheck) diff --git a/src/bin/pg_upgrade/TESTING b/src/bin/pg_upgrade/TESTING index e69874b42d..200ce9d92b 100644 --- a/src/bin/pg_upgrade/TESTING +++ b/src/bin/pg_upgrade/TESTING @@ -2,25 +2,22 @@ THE SHORT VERSION ----------------- On non-Windows machines, you can execute the testing process -described below by running +described below by running the following command in this directory: make check -in this directory. This will run the shell script test.sh, performing -an upgrade from the version in this source tree to a new instance of -the same version. -To test an upgrade from a different version, you must have a built -source tree for the old version as well as this version, and you -must have done "make install" for both versions. Then do: +This will run the TAP tests to run pg_upgrade, performing an upgrade +from the version in this source tree to a new instance of the same +version. -export oldsrc=...somewhere/postgresql (old version's source tree) -export oldbindir=...otherversion/bin (old version's installed bin dir) -export bindir=...thisversion/bin (this version's installed bin dir) -export libdir=...thisversion/lib (this version's installed lib dir) -sh test.sh - -In this case, you will have to manually eyeball the resulting dump -diff for version-specific differences, as explained below. +Testing an upgrade from a different version requires a dump to set up +the contents of this instance, with its set of binaries. The following +variables are available to control the test (see DETAILS below about +the creation of the dump): +export olddump=...somewhere/dump.sql (old version's dump) +export oldinstall=...otherversion/ (old version's install base path) +Finally, the tests can be done by running + make check DETAILS ------- @@ -29,61 +26,22 @@ The most effective way to test pg_upgrade, aside from testing on user data, is by upgrading the PostgreSQL regression database. This testing process first requires the creation of a valid regression -database dump. Such files contain most database features and are -specific to each major version of Postgres. +database dump that can be then used for $olddump. Such files contain +most database features and are specific to each major version of Postgres. -Here are the steps needed to create a regression database dump file: +Here are the steps needed to create a dump file: 1) Create and populate the regression database in the old cluster. This database can be created by running 'make installcheck' from - src/test/regress. - -2) Use pg_dump to dump out the regression database. Use the new - cluster's pg_dump on the old database to minimize whitespace - differences in the diff. - -3) Adjust the regression database dump file - - a) Perform the load/dump twice - This fixes problems with the ordering of COPY columns for - inherited tables. - - b) Change CREATE FUNCTION shared object paths to use '$libdir' - The old and new cluster will have different shared object paths. - - c) Fix any wrapping format differences - Commands like CREATE TRIGGER and ALTER TABLE sometimes have - differences. - - d) For pre-9.0, change CREATE OR REPLACE LANGUAGE to CREATE LANGUAGE - - e) For pre-9.0, remove 'regex_flavor' - - f) For pre-9.0, adjust extra_float_digits - Postgres 9.0 pg_dump uses extra_float_digits=-2 for pre-9.0 - databases, and extra_float_digits=-3 for >= 9.0 databases. - It is necessary to modify 9.0 pg_dump to always use -3, and - modify the pre-9.0 old server to accept extra_float_digits=-3. - -Once the dump is created, it can be repeatedly loaded into the old -database, upgraded, and dumped out of the new database, and then -compared to the original version. To test the dump file, perform these -steps: - -1) Create the old and new clusters in different directories. - -2) Copy the regression shared object files into the appropriate /lib - directory for old and new clusters. - -3) Create the regression database in the old server. - -4) Load the dump file created above into the regression database; - check for errors while loading. - -5) Upgrade the old database to the new major version, as outlined in - the pg_upgrade manual section. - -6) Use pg_dump to dump out the regression database in the new cluster. - -7) Diff the regression database dump file with the regression dump - file loaded into the old server. + src/test/regress using its source code tree. + +2) Use pg_dumpall to dump out the contents of the instance, including the + regression database, in the shape of a SQL file. This requires the *old* + cluster's pg_dumpall so as the dump created is compatible with the + version of the cluster it is dumped into. + +Once the dump is created, it can be repeatedly used with $olddump and +`make check`, that automates the dump of the old database, its upgrade, +the dump out of the new database and the comparison of the dumps between +the old and new databases. The contents of the dumps can also be manually +compared. diff --git a/src/bin/pg_upgrade/t/001_basic.pl b/src/bin/pg_upgrade/t/001_basic.pl new file mode 100644 index 0000000000..40458f10b6 --- /dev/null +++ b/src/bin/pg_upgrade/t/001_basic.pl @@ -0,0 +1,11 @@ +use strict; +use warnings; + +use PostgreSQL::Test::Utils; +use Test::More; + +program_help_ok('pg_upgrade'); +program_version_ok('pg_upgrade'); +program_options_handling_ok('pg_upgrade'); + +done_testing(); diff --git a/src/bin/pg_upgrade/t/002_pg_upgrade.pl b/src/bin/pg_upgrade/t/002_pg_upgrade.pl new file mode 100644 index 0000000000..877adfd2af --- /dev/null +++ b/src/bin/pg_upgrade/t/002_pg_upgrade.pl @@ -0,0 +1,323 @@ +# Set of tests for pg_upgrade, including cross-version checks. +use strict; +use warnings; + +use Cwd qw(abs_path); +use File::Basename qw(dirname); +use File::Compare; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +# Generate a database with a name made of a range of ASCII characters. +sub generate_db +{ + my ($node, $prefix, $from_char, $to_char, $suffix) = @_; + + my $dbname = $prefix; + for my $i ($from_char .. $to_char) + { + next if $i == 7 || $i == 10 || $i == 13; # skip BEL, LF, and CR + $dbname = $dbname . sprintf('%c', $i); + } + + $dbname .= $suffix; + $node->command_ok( + [ 'createdb', $dbname ], + "created database with ASCII characters from $from_char to $to_char"); +} + +# Filter the contents of a dump before its use in a content comparison. +# This returns the path to the filtered dump. +sub filter_dump +{ + my ($node, $tempdir, $dump_file_name) = @_; + my $dump_file = "$tempdir/$dump_file_name"; + my $dump_contents = slurp_file($dump_file); + + # Remove the comments. + $dump_contents =~ s/^\-\-.*//mgx; + + # there is no default_table_access_method in <=11 versions + $dump_contents =~ s/^SET\sdefault_table_access_method.*//mgx; + + # Locale setup has changed for collations in 15~. + $dump_contents =~ + s/(^CREATE\sCOLLATION\s.*?)\slocale\s=\s'und'/$1 locale = ''/mgx + if ($node->pg_version < 15); + + # Dumps taken from <= 11 use EXECUTE PROCEDURE. Replace it + # with EXECUTE FUNCTION. + $dump_contents =~ + s/(^CREATE\sTRIGGER\s.*?)\sEXECUTE\sPROCEDURE/$1 EXECUTE FUNCTION/mgx + if ($node->pg_version < 12); + + # Remove empty lines. + $dump_contents =~ s/^\n//mgx; + + my $dump_file_filtered = "$tempdir/${dump_file_name}_filter"; + open(my $dh, '>', $dump_file_filtered) + || die "opening $dump_file_filtered"; + print $dh $dump_contents; + close($dh); + + return $dump_file_filtered; +} + +# The test of pg_upgrade requires two clusters, an old one and a new one +# that gets upgraded. Before running the upgrade, a logical dump of the +# old cluster is taken, and a second logical dump of the new one is taken +# after the upgrade. The upgrade test passes if there are no differences +# in these two dumps. + +# Testing upgrades with an older version of PostgreSQL requires setting up +# two environment variables, as of: +# - "olddump", to point to a dump file that will be used to set up the old +# instance to upgrade from. +# - "oldinstall", to point to the installation path of the old cluster. +if ( (defined($ENV{olddump}) && !defined($ENV{oldinstall})) + || (!defined($ENV{olddump}) && defined($ENV{oldinstall}))) +{ + # Not all variables are defined, so leave and die if test is + # done with an older installation. + die "olddump or oldinstall is undefined"; +} + +# Temporary location for the dumps taken +my $tempdir = PostgreSQL::Test::Utils::tempdir; + +# Initialize node to upgrade +my $oldnode = PostgreSQL::Test::Cluster->get_new_node('old_node', + install_path => $ENV{oldinstall}); + +# To increase coverage of non-standard segment size and group access without +# increasing test runtime, run these tests with a custom setting. +# --allow-group-access and --wal-segsize have been added in v11. +my %node_params = (); +my $ver_with_newopts = 11; +my $oldver = $oldnode->{_pg_version}; +$node_params{extra} = [ '--wal-segsize', '1', '--allow-group-access' ] + if $oldver >= $ver_with_newopts; +$oldnode->init(%node_params); + +$oldnode->start; + +# The default location of the source code is the root of this directory. +my $srcdir = abs_path("../../.."); + +# Set up the data of the old instance with a dump or pg_regress. +if (defined($ENV{olddump})) +{ + # Use the dump specified. + my $olddumpfile = $ENV{olddump}; + die "no dump file found!" unless -e $olddumpfile; + + # Load the dump using the "postgres" database as "regression" does + # not exist yet, and we are done here. + $oldnode->command_ok([ 'psql', '-X', '-f', $olddumpfile, 'postgres' ], + 'loaded old dump file'); +} +else +{ + # Default is to use pg_regress to set up the old instance. + + # Create databases with names covering most ASCII bytes. The + # first name exercises backslashes adjacent to double quotes, a + # Windows special case. + generate_db($oldnode, 'regression\\"\\', 1, 45, '\\\\"\\\\\\'); + generate_db($oldnode, 'regression', 46, 90, ''); + generate_db($oldnode, 'regression', 91, 127, ''); + + # Grab any regression options that may be passed down by caller. + my $extra_opts = $ENV{EXTRA_REGRESS_OPTS} || ""; + + # --dlpath is needed to be able to find the location of regress.so + # and any libraries the regression tests require. + my $dlpath = dirname($ENV{REGRESS_SHLIB}); + + # --outputdir points to the path where to place the output files. + my $outputdir = $PostgreSQL::Test::Utils::tmp_check; + + # --inputdir points to the path of the input files. + my $inputdir = "$srcdir/src/test/regress"; + + mkdir $outputdir . "/sql"; + mkdir $outputdir . "/expected"; + mkdir $outputdir . "/testtablespace"; + + my $rc = + system($ENV{PG_REGRESS} + . " $extra_opts " + . "--dlpath=\"$dlpath\" " + . "--bindir= " + . "--host=" + . $oldnode->host . " " + . "--port=" + . $oldnode->port . " " + . "--schedule=$srcdir/src/test/regress/parallel_schedule " + . "--max-concurrent-tests=20 " + . "--inputdir=\"$inputdir\" " + . "--outputdir=\"$outputdir\""); + if ($rc != 0) + { + # Dump out the regression diffs file, if there is one + my $diffs = "$outputdir/regression.diffs"; + if (-e $diffs) + { + print "=== dumping $diffs ===\n"; + print slurp_file($diffs); + print "=== EOF ===\n"; + } + } + is($rc, 0, 'regression tests pass'); +} + +# Before dumping, get rid of objects not existing or not supported in later +# versions. This depends on the version of the old server used, and matters +# only if different major versions are used for the dump. +if (defined($ENV{oldinstall})) +{ + # Note that upgrade_adapt.sql from the new version is used, to + # cope with an upgrade to this version. + $oldnode->command_ok( + [ + 'psql', '-X', + '-f', "$srcdir/src/bin/pg_upgrade/upgrade_adapt.sql", + 'regression' + ], + 'ran adapt script'); +} + +# Initialize a new node for the upgrade. +my $newnode = PostgreSQL::Test::Cluster->new('new_node'); +$newnode->init(%node_params); +my $newbindir = $newnode->config_data('--bindir'); +my $oldbindir = $oldnode->config_data('--bindir'); + +# Take a dump before performing the upgrade as a base comparison. Note +# that we need to use pg_dumpall from the new node here. +my @dump_command = ( + 'pg_dumpall', '--no-sync', + '-d', $oldnode->connstr('postgres'), + '-f', "$tempdir/dump1.sql"); +# --extra-float-digits is needed when upgrading from a version older than 11. +push(@dump_command, '--extra-float-digits', '0') + if ($oldnode->pg_version < 12); +$newnode->command_ok(\@dump_command, 'dump before running pg_upgrade'); + +# After dumping, update references to the old source tree's regress.so +# to point to the new tree. +if (defined($ENV{oldinstall})) +{ + # First, fetch all the references to libraries that are not part + # of the default path $libdir. + my $output = $oldnode->safe_psql('regression', + "SELECT DISTINCT probin::text FROM pg_proc WHERE probin NOT LIKE '\$libdir%';" + ); + chomp($output); + my @libpaths = split("\n", $output); + + my $dump_data = slurp_file("$tempdir/dump1.sql"); + + my $newregresssrc = "$srcdir/src/test/regress"; + foreach (@libpaths) + { + my $libpath = $_; + $libpath = dirname($libpath); + $dump_data =~ s/$libpath/$newregresssrc/g; + } + + open my $fh, ">", "$tempdir/dump1.sql" or die "could not open dump file"; + print $fh $dump_data; + close $fh; + + # This replaces any references to the old tree's regress.so + # the new tree's regress.so. Any references that do *not* + # match $libdir are switched so as this request does not + # depend on the path of the old source tree. This is useful + # when using an old dump. Do the operation on all the databases + # that allow connections so as this includes the regression + # database and anything the user has set up. + $output = $oldnode->safe_psql('postgres', + "SELECT datname FROM pg_database WHERE datallowconn;"); + chomp($output); + my @datnames = split("\n", $output); + foreach (@datnames) + { + my $datname = $_; + $oldnode->safe_psql( + $datname, "UPDATE pg_proc SET probin = + regexp_replace(probin, '.*/', '$newregresssrc/') + WHERE probin NOT LIKE '\$libdir/%'"); + } +} + +# In a VPATH build, we'll be started in the source directory, but we want +# to run pg_upgrade in the build directory so that any files generated finish +# in it, like delete_old_cluster.{sh,bat}. +chdir ${PostgreSQL::Test::Utils::tmp_check}; + +# Upgrade the instance. +$oldnode->stop; +command_ok( + [ + 'pg_upgrade', '-d', $oldnode->data_dir, '-D', $newnode->data_dir, + '-b', $oldbindir, '-B', $newbindir, + '-s', $newnode->host, + '-p', $oldnode->port, '-P', $newnode->port + ], + 'run of pg_upgrade for new instance'); +$newnode->start; + +# Check if there are any logs coming from pg_upgrade, that would only be +# retained on failure. +my $log_path = $newnode->data_dir . "/pg_upgrade_output.d/log"; +if (-d $log_path) +{ + foreach my $log (glob("$log_path/*")) + { + note "=== contents of $log ===\n"; + print slurp_file($log); + print "=== EOF ===\n"; + } +} + +# Second dump from the upgraded instance. +@dump_command = ( + 'pg_dumpall', '--no-sync', '-d', $newnode->connstr('postgres'), + '-f', "$tempdir/dump2.sql"); +# --extra-float-digits is needed when upgrading from a version older than 11. +push(@dump_command, '--extra-float-digits', '0') + if ($oldnode->pg_version < 12); +$newnode->command_ok(\@dump_command, 'dump after running pg_upgrade'); + +# Filter the contents of the dumps from the old version of any contents. +my $dump1_filtered = "$tempdir/dump1.sql"; +my $dump2_filtered = "$tempdir/dump2.sql"; + +# No need to apply filters on the dumps if working on the same version. +if ($oldnode->pg_version != $newnode->pg_version) +{ + $dump1_filtered = filter_dump($oldnode, $tempdir, "dump1.sql"); + $dump2_filtered = filter_dump($newnode, $tempdir, "dump2.sql"); +} + +# Compare the two dumps, there should be no differences. +my $compare_res = compare($dump1_filtered, $dump2_filtered); +is($compare_res, 0, 'old and new dumps match after pg_upgrade'); + +# Provide more context if the dumps do not match. +if ($compare_res != 0) +{ + my ($stdout, $stderr) = + run_command([ 'diff', $dump1_filtered, $dump2_filtered ]); + print "=== diff of $dump1_filtered and $dump2_filtered\n"; + print "=== stdout ===\n"; + print $stdout; + print "=== stderr ===\n"; + print $stderr; + print "=== EOF ===\n"; +} + +done_testing(); diff --git a/src/bin/pg_upgrade/test.sh b/src/bin/pg_upgrade/test.sh deleted file mode 100644 index f353e565b5..0000000000 --- a/src/bin/pg_upgrade/test.sh +++ /dev/null @@ -1,282 +0,0 @@ -#!/bin/sh - -# src/bin/pg_upgrade/test.sh -# -# Test driver for pg_upgrade. Initializes a new database cluster, -# runs the regression tests (to put in some data), runs pg_dumpall, -# runs pg_upgrade, runs pg_dumpall again, compares the dumps. -# -# Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group -# Portions Copyright (c) 1994, Regents of the University of California - -set -e - -: ${MAKE=make} - -# Guard against parallel make issues (see comments in pg_regress.c) -unset MAKEFLAGS -unset MAKELEVEL - -# Run a given "initdb" binary and overlay the regression testing -# authentication configuration. -standard_initdb() { - # To increase coverage of non-standard segment size and group access - # without increasing test runtime, run these tests with a custom setting. - # Also, specify "-A trust" explicitly to suppress initdb's warning. - # --allow-group-access and --wal-segsize have been added in v11. - "$1" -N --wal-segsize 1 --allow-group-access -A trust - if [ -n "$TEMP_CONFIG" -a -r "$TEMP_CONFIG" ] - then - cat "$TEMP_CONFIG" >> "$PGDATA/postgresql.conf" - fi - ../../test/regress/pg_regress --config-auth "$PGDATA" -} - -# What flavor of host are we on? -# Treat MINGW* (msys1) and MSYS* (msys2) the same. -testhost=`uname -s | sed 's/^MSYS/MINGW/'` - -# Establish how the server will listen for connections -case $testhost in - MINGW*) - LISTEN_ADDRESSES="localhost" - PG_REGRESS_SOCKET_DIR="" - PGHOST=localhost - ;; - *) - LISTEN_ADDRESSES="" - # Select a socket directory. The algorithm is from the "configure" - # script; the outcome mimics pg_regress.c:make_temp_sockdir(). - if [ x"$PG_REGRESS_SOCKET_DIR" = x ]; then - set +e - dir=`(umask 077 && - mktemp -d /tmp/pg_upgrade_check-XXXXXX) 2>/dev/null` - if [ ! -d "$dir" ]; then - dir=/tmp/pg_upgrade_check-$$-$RANDOM - (umask 077 && mkdir "$dir") - if [ ! -d "$dir" ]; then - echo "could not create socket temporary directory in \"/tmp\"" - exit 1 - fi - fi - set -e - PG_REGRESS_SOCKET_DIR=$dir - trap 'rm -rf "$PG_REGRESS_SOCKET_DIR"' 0 - trap 'exit 3' 1 2 13 15 - fi - PGHOST=$PG_REGRESS_SOCKET_DIR - ;; -esac - -POSTMASTER_OPTS="-F -c listen_addresses=\"$LISTEN_ADDRESSES\" -k \"$PG_REGRESS_SOCKET_DIR\"" -export PGHOST - -# don't rely on $PWD here, as old shells don't set it -temp_root=`pwd`/tmp_check -rm -rf "$temp_root" -mkdir "$temp_root" - -: ${oldbindir=$bindir} - -: ${oldsrc=../../..} -oldsrc=`cd "$oldsrc" && pwd` -newsrc=`cd ../../.. && pwd` - -# We need to make pg_regress use psql from the desired installation -# (likely a temporary one), because otherwise the installcheck run -# below would try to use psql from the proper installation directory -# of the target version, which might be outdated or not exist. But -# don't override anything else that's already in EXTRA_REGRESS_OPTS. -EXTRA_REGRESS_OPTS="$EXTRA_REGRESS_OPTS --bindir='$oldbindir'" -export EXTRA_REGRESS_OPTS - -# While in normal cases this will already be set up, adding bindir to -# path allows test.sh to be invoked with different versions as -# described in ./TESTING -PATH=$bindir:$PATH -export PATH - -BASE_PGDATA="$temp_root/data" -PGDATA="${BASE_PGDATA}.old" -export PGDATA - -# Send installcheck outputs to a private directory. This avoids conflict when -# check-world runs pg_upgrade check concurrently with src/test/regress check. -# To retrieve interesting files after a run, use pattern tmp_check/*/*.diffs. -outputdir="$temp_root/regress" -EXTRA_REGRESS_OPTS="$EXTRA_REGRESS_OPTS --outputdir=$outputdir" -export EXTRA_REGRESS_OPTS -mkdir "$outputdir" - -# pg_regress --make-tablespacedir would take care of that in 14~, but this is -# still required for older versions where this option is not supported. -if [ "$newsrc" != "$oldsrc" ]; then - mkdir "$outputdir"/testtablespace - mkdir "$outputdir"/sql - mkdir "$outputdir"/expected -fi - -logdir=`pwd`/log -rm -rf "$logdir" -mkdir "$logdir" - -# Clear out any environment vars that might cause libpq to connect to -# the wrong postmaster (cf pg_regress.c) -# -# Some shells, such as NetBSD's, return non-zero from unset if the variable -# is already unset. Since we are operating under 'set -e', this causes the -# script to fail. To guard against this, set them all to an empty string first. -PGDATABASE=""; unset PGDATABASE -PGUSER=""; unset PGUSER -PGSERVICE=""; unset PGSERVICE -PGSSLMODE=""; unset PGSSLMODE -PGREQUIRESSL=""; unset PGREQUIRESSL -PGCONNECT_TIMEOUT=""; unset PGCONNECT_TIMEOUT -PGHOSTADDR=""; unset PGHOSTADDR - -# Select a non-conflicting port number, similarly to pg_regress.c -PG_VERSION_NUM=`grep '#define PG_VERSION_NUM' "$newsrc"/src/include/pg_config.h | awk '{print $3}'` -PGPORT=`expr $PG_VERSION_NUM % 16384 + 49152` -export PGPORT - -i=0 -while psql -X postgres /dev/null -do - i=`expr $i + 1` - if [ $i -eq 16 ] - then - echo port $PGPORT apparently in use - exit 1 - fi - PGPORT=`expr $PGPORT + 1` - export PGPORT -done - -# buildfarm may try to override port via EXTRA_REGRESS_OPTS ... -EXTRA_REGRESS_OPTS="$EXTRA_REGRESS_OPTS --port=$PGPORT" -export EXTRA_REGRESS_OPTS - -standard_initdb "$oldbindir"/initdb -"$oldbindir"/pg_ctl start -l "$logdir/postmaster1.log" -o "$POSTMASTER_OPTS" -w - -# Create databases with names covering the ASCII bytes other than NUL, BEL, -# LF, or CR. BEL would ring the terminal bell in the course of this test, and -# it is not otherwise a special case. PostgreSQL doesn't support the rest. -dbname1=`awk 'BEGIN { for (i= 1; i < 46; i++) - if (i != 7 && i != 10 && i != 13) printf "%c", i }' "$temp_root"/dump1.sql - fi -else - make_installcheck_status=$? -fi -"$oldbindir"/pg_ctl -m fast stop -if [ -n "$createdb_status" ]; then - exit 1 -fi -if [ -n "$make_installcheck_status" ]; then - exit 1 -fi -if [ -n "$psql_fix_sql_status" ]; then - exit 1 -fi -if [ -n "$pg_dumpall1_status" ]; then - echo "pg_dumpall of pre-upgrade database cluster failed" - exit 1 -fi - -PGDATA="$BASE_PGDATA" - -standard_initdb 'initdb' - -pg_upgrade $PG_UPGRADE_OPTS -d "${PGDATA}.old" -D "$PGDATA" -b "$oldbindir" -p "$PGPORT" -P "$PGPORT" - -# make sure all directories and files have group permissions, on Unix hosts -# Windows hosts don't support Unix-y permissions. -case $testhost in - MINGW*|CYGWIN*) ;; - *) if [ `find "$PGDATA" -type f ! -perm 640 | wc -l` -ne 0 ]; then - echo "files in PGDATA with permission != 640"; - exit 1; - fi ;; -esac - -case $testhost in - MINGW*|CYGWIN*) ;; - *) if [ `find "$PGDATA" -type d ! -perm 750 | wc -l` -ne 0 ]; then - echo "directories in PGDATA with permission != 750"; - exit 1; - fi ;; -esac - -pg_ctl start -l "$logdir/postmaster2.log" -o "$POSTMASTER_OPTS" -w - -pg_dumpall $extra_dump_options --no-sync \ - -f "$temp_root"/dump2.sql || pg_dumpall2_status=$? -pg_ctl -m fast stop - -if [ -n "$pg_dumpall2_status" ]; then - echo "pg_dumpall of post-upgrade database cluster failed" - exit 1 -fi - -case $testhost in - MINGW*) MSYS2_ARG_CONV_EXCL=/c cmd /c delete_old_cluster.bat ;; - *) sh ./delete_old_cluster.sh ;; -esac - -if diff "$temp_root"/dump1.sql "$temp_root"/dump2.sql >/dev/null; then - echo PASSED - exit 0 -else - echo "Files $temp_root/dump1.sql and $temp_root/dump2.sql differ" - echo "dumps were not identical" - exit 1 -fi diff --git a/src/bin/pg_upgrade/upgrade_adapt.sql b/src/bin/pg_upgrade/upgrade_adapt.sql index 27c4c7fd01..07c86791af 100644 --- a/src/bin/pg_upgrade/upgrade_adapt.sql +++ b/src/bin/pg_upgrade/upgrade_adapt.sql @@ -46,6 +46,7 @@ CREATE AGGREGATE array_larger_accum (anyarray) ( DROP OPERATOR @#@ (NONE, bigint); CREATE OPERATOR @#@ (PROCEDURE = factorial, RIGHTARG = bigint); +DROP OPERATOR IF EXISTS public.=> (pg_catalog.int8, NONE); \endif -- Objects last appearing in 9.6. @@ -84,8 +85,8 @@ DO $stmt$ \if :oldpgversion_le13 -- Until v10, operators could only be dropped one at a time, so be careful -- to stick with one command for each drop here. -DROP OPERATOR public.#@# (pg_catalog.int8, NONE); -DROP OPERATOR public.#%# (pg_catalog.int8, NONE); -DROP OPERATOR public.!=- (pg_catalog.int8, NONE); -DROP OPERATOR public.#@%# (pg_catalog.int8, NONE); +DROP OPERATOR IF EXISTS public.#@# (pg_catalog.int8, NONE); +DROP OPERATOR IF EXISTS public.#%# (pg_catalog.int8, NONE); +DROP OPERATOR IF EXISTS public.!=- (pg_catalog.int8, NONE); +DROP OPERATOR IF EXISTS public.#@%# (pg_catalog.int8, NONE); \endif diff --git a/src/test/perl/PostgresNode.pm b/src/test/perl/PostgresNode.pm index 4b17048578..c9a12949fd 100644 --- a/src/test/perl/PostgresNode.pm +++ b/src/test/perl/PostgresNode.pm @@ -352,6 +352,45 @@ sub backup_dir =pod +=item $node->pg_version() + +The version number for the node, from PostgreSQL::Version. + +=cut + +sub pg_version +{ + my ($self) = @_; + return $self->{_pg_version}; +} + +=pod + +=item $node->config_data($option) + +Return a string holding configuration data from pg_config, with $option +being the option switch used with the pg_config command. + +=cut + +sub config_data +{ + my ($self, $option) = @_; + local %ENV = $self->_get_env(); + + my ($stdout, $stderr); + my $result = + IPC::Run::run [ $self->installed_command('pg_config'), $option ], + '>', \$stdout, '2>', \$stderr + or die "could not execute pg_config"; + chomp($stdout); + $stdout =~ s/\r$//; + + return $stdout; +} + +=pod + =item $node->info() Return a string containing human-readable diagnostic information (paths, etc) @@ -455,8 +494,14 @@ sub init mkdir $self->backup_dir; mkdir $self->archive_dir; - TestLib::system_or_bail('initdb', '-D', $pgdata, '-A', 'trust', '-N', - @{ $params{extra} }); + my $ver_with_no_sync = 9.3; + my $ver = $self->{_pg_version}; + my @initdb_opts = ('-D', $pgdata, + '-A', 'trust'); + push @initdb_opts, '-N' + if $ver >= $ver_with_no_sync; + + TestLib::system_or_bail('initdb', @initdb_opts, @{ $params{extra} }); TestLib::system_or_bail($ENV{PG_REGRESS}, '--config-auth', $pgdata, @{ $params{auth_extra} }); @@ -466,8 +511,11 @@ sub init print $conf "restart_after_crash = off\n"; print $conf "log_line_prefix = '%m [%p] %q%a '\n"; print $conf "log_statement = all\n"; - print $conf "log_replication_commands = on\n"; - print $conf "wal_retrieve_retry_interval = '500ms'\n"; + my $ver_with_extra_opts = 9.5; + print $conf "log_replication_commands = on\n" + if $ver >= $ver_with_extra_opts; + print $conf "wal_retrieve_retry_interval = '500ms'\n" + if $ver >= $ver_with_extra_opts; # If a setting tends to affect whether tests pass or fail, print it after # TEMP_CONFIG. Otherwise, print it before TEMP_CONFIG, thereby permitting @@ -507,14 +555,29 @@ sub init } print $conf "port = $port\n"; + my $ver_with_unix_socket_dirs = 9.3; if ($use_tcp) { - print $conf "unix_socket_directories = ''\n"; + if ($ver >= $ver_with_unix_socket_dirs) + { + print $conf "unix_socket_directories = ''\n"; + } + else + { + print $conf "unix_socket_directory = ''\n"; + } print $conf "listen_addresses = '$host'\n"; } else { - print $conf "unix_socket_directories = '$host'\n"; + if ($ver >= $ver_with_unix_socket_dirs) + { + print $conf "unix_socket_directories = '$host'\n"; + } + else + { + print $conf "unix_socket_directory = '$host'\n"; + } print $conf "listen_addresses = ''\n"; } close $conf; @@ -815,10 +878,17 @@ sub start # connections in confusing ways. local %ENV = $self->_get_env(PGAPPNAME => undef); - # Note: We set the cluster_name here, not in postgresql.conf (in - # sub init) so that it does not get copied to standbys. - $ret = TestLib::system_log('pg_ctl', '-D', $self->data_dir, '-l', - $self->logfile, '-o', "--cluster-name=$name", 'start'); + # Note: We set the cluster_name if supported here, not in postgresql.conf + # (in sub init) so that it does not get copied to standbys. + # -w is now the default but having it here does no harm and helps + # compatibility with older versions. + my $ver_with_cluster_name = 9.5; + my $ver = $self->{_pg_version}; + my @pg_ctl_opts = ('-w', '-D', $self->data_dir, + '-l', $self->logfile); + push @pg_ctl_opts, '-o', "--cluster-name=$name" + if $ver >= $ver_with_cluster_name; + $ret = TestLib::system_log('pg_ctl', @pg_ctl_opts, 'start'); if ($ret != 0) { @@ -951,7 +1021,9 @@ sub restart print "### Restarting node \"$name\"\n"; - TestLib::system_or_bail('pg_ctl', '-D', $pgdata, '-l', $logfile, + # -w is now the default but having it here does no harm and helps + # compatibility with older versions. + TestLib::system_or_bail('pg_ctl', '-w', '-D', $pgdata, '-l', $logfile, 'restart'); $self->_update_pid(1); @@ -1265,42 +1337,44 @@ sub get_new_node # sub _set_pg_version { - my ($self) = @_; - my $inst = $self->{_install_path}; - my $pg_config = "pg_config"; - - if (defined $inst) - { - # If the _install_path is invalid, our PATH variables might find an - # unrelated pg_config executable elsewhere. Sanity check the - # directory. - BAIL_OUT("directory not found: $inst") - unless -d $inst; - - # If the directory exists but is not the root of a postgresql - # installation, or if the user configured using - # --bindir=$SOMEWHERE_ELSE, we're not going to find pg_config, so - # complain about that, too. - $pg_config = "$inst/bin/pg_config"; - BAIL_OUT("pg_config not found: $pg_config") - unless -e $pg_config - or ($TestLib::windows_os and -e "$pg_config.exe"); - BAIL_OUT("pg_config not executable: $pg_config") - unless $TestLib::windows_os or -x $pg_config; - - # Leave $pg_config install_path qualified, to be sure we get the right - # version information, below, or die trying - } - - local %ENV = $self->_get_env(); - - # We only want the version field - my $version_line = qx{$pg_config --version}; - BAIL_OUT("$pg_config failed: $!") if $?; - - $self->{_pg_version} = PostgresVersion->new($version_line); - - BAIL_OUT("could not parse pg_config --version output: $version_line") + my ($self) = @_; + my $inst = $self->{_install_path}; + my $pg_config = "pg_config"; + + if (defined $inst) + { + # If the _install_path is invalid, our PATH variables might find an + # unrelated pg_config executable elsewhere. Sanity check the + # directory. + BAIL_OUT("directory not found: $inst") + unless -d $inst; + + # If the directory exists but is not the root of a postgresql + # installation, or if the user configured using + # --bindir=$SOMEWHERE_ELSE, we're not going to find pg_config, so + # complain about that, too. + $pg_config = "$inst/bin/pg_config"; + BAIL_OUT("pg_config not found: $pg_config") + unless -e $pg_config; + BAIL_OUT("pg_config not executable: $pg_config") + unless -x $pg_config; + + # Leave $pg_config install_path qualified, to be sure we get the right + # version information, below, or die trying + } + + local %ENV = $self->_get_env(); + + # We only want the version field + open my $fh, "-|", $pg_config, "--version" + or + BAIL_OUT("$pg_config failed: $!"); + my $version_line = <$fh>; + close $fh or die; + + $self->{_pg_version} = PostgresVersion->new($version_line); + + BAIL_OUT("could not parse pg_config --version output: $version_line") unless defined $self->{_pg_version}; } @@ -1324,7 +1398,7 @@ sub _set_pg_version # a common parent directory. sub _get_env { - my $self = shift; + my $self = shift; my %inst_env = (%ENV, PGHOST => $self->{_host}, PGPORT => $self->{_port}); # the remaining arguments are modifications to make to the environment my %mods = (@_); diff --git a/src/tools/msvc/vcregress.pl b/src/tools/msvc/vcregress.pl index 2b7efe29e4..755092bb22 100644 --- a/src/tools/msvc/vcregress.pl +++ b/src/tools/msvc/vcregress.pl @@ -319,6 +319,10 @@ sub bincheck foreach my $dir (@bin_dirs) { next unless -d "$dir/t"; + # Do not consider pg_upgrade, as it is handled by + # upgradecheck. + next if ($dir =~ "/pg_upgrade/"); + my $status = tap_check($dir); $mstat ||= $status; } @@ -626,89 +630,9 @@ sub generate_db sub upgradecheck { - my $status; - my $cwd = getcwd(); - - # Much of this comes from the pg_upgrade test.sh script, - # but it only covers the --install case, and not the case - # where the old and new source or bin dirs are different. - # i.e. only this version to this version check. That's - # what pg_upgrade's "make check" does. - - $ENV{PGHOST} = 'localhost'; - $ENV{PGPORT} ||= 50432; - my $tmp_root = "$topdir/src/bin/pg_upgrade/tmp_check"; - rmtree($tmp_root); - mkdir $tmp_root || die $!; - my $upg_tmp_install = "$tmp_root/install"; # unshared temp install - print "Setting up temp install\n\n"; - Install($upg_tmp_install, "all", $config); - - # Install does a chdir, so change back after that - chdir $cwd; - my ($bindir, $libdir, $oldsrc, $newsrc) = - ("$upg_tmp_install/bin", "$upg_tmp_install/lib", $topdir, $topdir); - $ENV{PATH} = "$bindir;$ENV{PATH}"; - my $data = "$tmp_root/data"; - $ENV{PGDATA} = "$data.old"; - my $outputdir = "$tmp_root/regress"; - my @EXTRA_REGRESS_OPTS = ("--outputdir=$outputdir"); - mkdir "$outputdir" || die $!; - - my $logdir = "$topdir/src/bin/pg_upgrade/log"; - rmtree($logdir); - mkdir $logdir || die $!; - print "\nRunning initdb on old cluster\n\n"; - standard_initdb() or exit 1; - print "\nStarting old cluster\n\n"; - my @args = ('pg_ctl', 'start', '-l', "$logdir/postmaster1.log"); - system(@args) == 0 or exit 1; - - print "\nCreating databases with names covering most ASCII bytes\n\n"; - generate_db("\\\"\\", 1, 45, "\\\\\"\\\\\\"); - generate_db('', 46, 90, ''); - generate_db('', 91, 127, ''); - - print "\nSetting up data for upgrading\n\n"; - installcheck_internal('parallel', @EXTRA_REGRESS_OPTS); - - # now we can chdir into the source dir - chdir "$topdir/src/bin/pg_upgrade"; - print "\nDumping old cluster\n\n"; - @args = ('pg_dumpall', '-f', "$tmp_root/dump1.sql"); - system(@args) == 0 or exit 1; - print "\nStopping old cluster\n\n"; - system("pg_ctl stop") == 0 or exit 1; - $ENV{PGDATA} = "$data"; - print "\nSetting up new cluster\n\n"; - standard_initdb() or exit 1; - print "\nRunning pg_upgrade\n\n"; - @args = ('pg_upgrade', '-d', "$data.old", '-D', $data, '-b', $bindir); - system(@args) == 0 or exit 1; - print "\nStarting new cluster\n\n"; - @args = ('pg_ctl', '-l', "$logdir/postmaster2.log", 'start'); - system(@args) == 0 or exit 1; - print "\nDumping new cluster\n\n"; - @args = ('pg_dumpall', '-f', "$tmp_root/dump2.sql"); - system(@args) == 0 or exit 1; - print "\nStopping new cluster\n\n"; - system("pg_ctl stop") == 0 or exit 1; - print "\nDeleting old cluster\n\n"; - system(".\\delete_old_cluster.bat") == 0 or exit 1; - print "\nComparing old and new cluster dumps\n\n"; - - @args = ('diff', '-q', "$tmp_root/dump1.sql", "$tmp_root/dump2.sql"); - system(@args); - $status = $?; - if (!$status) - { - print "PASSED\n"; - } - else - { - print "dumps not identical!\n"; - exit(1); - } + InstallTemp(); + my $mstat = tap_check("$topdir/src/bin/pg_upgrade"); + exit $mstat if $mstat; return; }