From 3d5b24691517c1aac4b49728abb122c66a4e33be Mon Sep 17 00:00:00 2001 From: Kyotaro Horiguchi Date: Mon, 28 Mar 2022 16:29:04 +0900 Subject: [PATCH 1/2] Tentative test for tsp replay fix --- src/test/perl/PostgresNode.pm | 342 +++++++++++++++++++++- src/test/recovery/t/011_crash_recovery.pl | 108 ++++++- 2 files changed, 447 insertions(+), 3 deletions(-) diff --git a/src/test/perl/PostgresNode.pm b/src/test/perl/PostgresNode.pm index 7b2ec29bb7..88fa08b61d 100644 --- a/src/test/perl/PostgresNode.pm +++ b/src/test/perl/PostgresNode.pm @@ -104,6 +104,8 @@ use TestLib (); use Time::HiRes qw(usleep); use Scalar::Util qw(blessed); +my $windows_os = 0; + our @EXPORT = qw( get_new_node get_free_port @@ -323,6 +325,64 @@ sub archive_dir =pod +=item $node->tablespace_storage([, nocreate]) + +Diretory to store tablespace directories. +If nocreate is true, returns undef if not yet created. + +=cut + +sub tablespace_storage +{ + my ($self, $nocreate) = @_; + + if (!defined $self->{_tsproot}) + { + # tablespace is not used, return undef if nocreate is specified. + return undef if ($nocreate); + + # create and remember the tablespae root directotry. + $self->{_tsproot} = TestLib::tempdir_short(); + } + + return $self->{_tsproot}; +} + +=pod + +=item $node->tablespaces() + +Returns a hash from tablespace OID to tablespace directory name. For +example, an oid 16384 pointing to /tmp/jWAhkT_fs0/ts1 is stored as +$hash{16384} = "ts1". + +=cut + +sub tablespaces +{ + my ($self) = @_; + my $pg_tblspc = $self->data_dir . '/pg_tblspc'; + my %ret; + + # return undef if no tablespace is used + return undef if (!defined $self->tablespace_storage(1)); + + # collect tablespace entries in pg_tblspc directory + opendir(my $dir, $pg_tblspc); + while (my $oid = readdir($dir)) + { + next if ($oid !~ /^([0-9]+)$/); + my $linkpath = "$pg_tblspc/$oid"; + my $tsppath = dir_readlink($linkpath); + $ret{$oid} = File::Basename::basename($tsppath); + } + closedir($dir); + + return %ret; +} + +=pod + =item $node->backup_dir() The output path for backups taken with $node->backup() @@ -338,6 +398,77 @@ sub backup_dir =pod +=item $node->backup_tablespace_storage_path(backup_name) + +Returns tablespace location path for backup_name. +Retuns the parent directory if backup_name is not given. + +=cut + +sub backup_tablespace_storage_path +{ + my ($self, $backup_name) = @_; + my $dir = $self->backup_dir . '/__tsps'; + + $dir .= "/$backup_name" if (defined $backup_name); + + return $dir; +} + +=pod + +=item $node->backup_create_tablespace_storage(backup_name) + +Create tablespace location directory for backup_name if not yet. +Create the parent tablespace storage that holds all location +directories if backup_name is not supplied. + +=cut + +sub backup_create_tablespace_storage +{ + my ($self, $backup_name) = @_; + my $dir = $self->backup_tablespace_storage_path($backup_name); + + File::Path::make_path $dir if (! -d $dir); +} + +=pod + +=item $node->backup_tablespaces(backup_name) + +Returns a reference to hash from tablespace OID to tablespace +directory name of tablespace directory that the specified backup has. +For example, an oid 16384 pointing to ../tsps/backup1/ts1 is stored as +$hash{16384} = "ts1". + +=cut + +sub backup_tablespaces +{ + my ($self, $backup_name) = @_; + my $pg_tblspc = $self->backup_dir . '/' . $backup_name . '/pg_tblspc'; + my %ret; + + #return undef if this backup holds no tablespaces + return undef if (! -d $self->backup_tablespace_storage_path($backup_name)); + + # scan pg_tblspc directory of the backup + opendir(my $dir, $pg_tblspc); + while (my $oid = readdir($dir)) + { + next if ($oid !~ /^([0-9]+)$/); + my $linkpath = "$pg_tblspc/$oid"; + my $tsppath = dir_readlink($linkpath); + $ret{$oid} = File::Basename::basename($tsppath); + } + closedir($dir); + + return \%ret; +} + +=pod + =item $node->info() Return a string containing human-readable diagnostic information (paths, etc) @@ -354,6 +485,7 @@ sub info print $fh "Data directory: " . $self->data_dir . "\n"; print $fh "Backup directory: " . $self->backup_dir . "\n"; print $fh "Archive directory: " . $self->archive_dir . "\n"; + print $fh "Tablespace directory: " . $self->tablespace_storage . "\n"; print $fh "Connection string: " . $self->connstr . "\n"; print $fh "Log file: " . $self->logfile . "\n"; close $fh or die; @@ -536,6 +668,43 @@ sub append_conf =pod +=item $node->new_tablespace(name) + +Create a tablespace directory with the name then returns the path. + +=cut + +sub new_tablespace +{ + my ($self, $name) = @_; + + my $path = $self->tablespace_storage . '/' . $name; + + die "tablespace \"$name\" already exists" if (!mkdir($path)); + + return $path; +} + +=pod + +=item $node->tablespace_dir(name) + +Return the path of the existing tablespace with the name. + +=cut + +sub tablespace_dir +{ + my ($self, $name) = @_; + + my $path = $self->tablespace_storage . '/' . $name; + return undef if (!-d $path); + + return $path; +} + +=pod + =item $node->backup(backup_name) Create a hot backup with B in subdirectory B of @@ -555,13 +724,54 @@ sub backup my ($self, $backup_name, %params) = @_; my $backup_path = $self->backup_dir . '/' . $backup_name; my $name = $self->name; + my @tsp_maps; + + # Build tablespace mappings. We once let pg_basebackup copy + # tablespaces into temporary tablespace storage with a short name + # so that we can work on pathnames that fit our tar format which + # pg_basebackup depends on. + my $map_src_root = $self->tablespace_storage(1); + my $backup_tmptsp_root = TestLib::tempdir_short(); + my %tsps = $self->tablespaces(); + foreach my $tspname (values %tsps) + { + my $src = "$map_src_root/$tspname"; + my $dst = "$backup_tmptsp_root/$tspname"; + push(@tsp_maps, "--tablespace-mapping=$src=$dst"); + } print "# Taking pg_basebackup $backup_name from node \"$name\"\n"; TestLib::system_or_bail( 'pg_basebackup', '-D', $backup_path, '-h', $self->host, '-p', $self->port, '--checkpoint', 'fast', '--no-sync', + @tsp_maps, @{ $params{backup_options} }); + + # Move the tablespaces from temporary storage into backup + # directory, unless the backup is in tar mode. + if (%tsps && ! -f "$backup_path/base.tar") + { + $self->backup_create_tablespace_storage(); + RecursiveCopy::copypath( + $backup_tmptsp_root, + $self->backup_tablespace_storage_path($backup_name)); + # delete the temporary directory right away + rmtree $backup_tmptsp_root; + + # Fix tablespace symlinks. This is not necessarily required + # in backups but keep them consistent. + my $linkdst_root = "$backup_path/pg_tblspc"; + my $linksrc_root = $self->backup_tablespace_storage_path($backup_name); + foreach my $oid (keys %tsps) + { + my $tspdst = "$linkdst_root/$oid"; + my $tspsrc = "$linksrc_root/" . $tsps{$oid}; + unlink $tspdst; + dir_symlink($tspsrc, $tspdst); + } + } + print "# Backup finished\n"; return; } @@ -623,11 +833,32 @@ sub _backup_fs RecursiveCopy::copypath( $self->data_dir, $backup_path, + # Skipping some files and tablespace symlinks filterfn => sub { my $src = shift; - return ($src ne 'log' and $src ne 'postmaster.pid'); + return ($src ne 'log' and $src ne 'postmaster.pid' and + $src !~ m!^pg_tblspc/[0-9]+$!); }); + # Copy tablespaces if any + my %tsps = $self->tablespaces(); + if (%tsps) + { + $self->backup_create_tablespace_storage(); + RecursiveCopy::copypath( + $self->tablespace_storage, + $self->backup_tablespace_storage_path($backup_name)); + + my $linkdst_root = $backup_path . '/pg_tblspc'; + my $linksrc_root = $self->backup_tablespace_storage_path($backup_name); + foreach my $oid (keys %tsps) + { + my $tspdst = "$linkdst_root/$oid"; + my $tspsrc = "$linksrc_root/" . $tsps{$oid}; + dir_symlink($tspsrc, $tspdst); + } + } + if ($hot) { @@ -645,6 +876,80 @@ sub _backup_fs +=pod + +=item dir_symlink(oldname, newname) + +Portably create a symlink for a directory. On Windows this creates a junction +point. Elsewhere it just calls perl's builtin symlink. + +=cut + +sub dir_symlink +{ + my $oldname = shift; + my $newname = shift; + if ($windows_os) + { + $oldname =~ s,/,\\,g; + $newname =~ s,/,\\,g; + my $cmd = qq{mklink /j "$newname" "$oldname"}; + if ($Config{osname} eq 'msys') + { + # need some indirection on msys + $cmd = qq{echo '$cmd' | \$COMSPEC /Q}; + } + system($cmd); + } + else + { + symlink $oldname, $newname; + } + die "No $newname" unless -e $newname; +} + +=pod + +=item dir_readlink(name) + +Portably read a symlink for a directory. On Windows this reads a junction +point. Elsewhere it just calls perl's builtin readlink. + +=cut + +sub dir_readlink +{ + my $name = shift; + if ($windows_os) + { + $name .= '/..'; + $name =~ s,/,\\,g; + # Split the path into parent directory and link name + die "invalid path spec: $name" if ($name !~ m!^(.*)\\([^\\]+)\\?$!); + my ($dir, $fname) = ($1, $2); + my $cmd = qq{cmd /c "dir /A:L $dir"}; + if ($Config{osname} eq 'msys') + { + # need some indirection on msys + $cmd = qq{echo '$cmd' | \$COMSPEC /Q}; + } + + my $result; + foreach my $l (split /[\r\n]+/, `$cmd`) + { + $result = $1 if ($l =~ m/\W+$fname \[(.*)\]/) + } + die "junction $name not found" if (!defined $result); + + $name =~ s,\\,/,g; + return $result; + } + else + { + return readlink $name; + } +} + =pod =item $node->init_from_backup(root_node, backup_name) @@ -689,7 +994,40 @@ sub init_from_backup my $data_path = $self->data_dir; rmdir($data_path); - RecursiveCopy::copypath($backup_path, $data_path); + + RecursiveCopy::copypath( + $backup_path, + $data_path, + # Skipping tablespace symlinks + filterfn => sub { + my $src = shift; + return ($src !~ m!^pg_tblspc/[0-9]+$!); + }); + + # Copy tablespaces if any + my $tsps = $root_node->backup_tablespaces($backup_name); + + if ($tsps) + { + my $tsp_src = $root_node->backup_tablespace_storage_path($backup_name); + my $tsp_dst = $self->tablespace_storage(); + my $linksrc_root = $data_path . '/pg_tblspc'; + + # copypath() rejects to copy into existing directory. + # Copy individual directories in the storage. + foreach my $oid (keys %{$tsps}) + { + my $tsp = ${$tsps}{$oid}; + my $tspsrc = "$tsp_src/$tsp"; + my $tspdst = "$tsp_dst/$tsp"; + RecursiveCopy::copypath($tspsrc, $tspdst); + + # Create tablespace symlink for this tablespace + my $linkdst = "$linksrc_root/$oid"; + dir_symlink($tspdst, $linkdst); + } + } + chmod(0700, $data_path); # Base configuration for this node diff --git a/src/test/recovery/t/011_crash_recovery.pl b/src/test/recovery/t/011_crash_recovery.pl index 5dc52412ca..30aaf763e5 100644 --- a/src/test/recovery/t/011_crash_recovery.pl +++ b/src/test/recovery/t/011_crash_recovery.pl @@ -15,7 +15,7 @@ if ($Config{osname} eq 'MSWin32') } else { - plan tests => 3; + plan tests => 5; } my $node = get_new_node('master'); @@ -66,3 +66,109 @@ is($node->safe_psql('postgres', qq[SELECT txid_status('$xid');]), 'aborted', 'xid is aborted after crash'); $tx->kill_kill; + +my $node_primary = get_new_node('primary2'); +$node_primary->init(allows_streaming => 1); +$node_primary->start; +my $dropme_ts_primary1 = $node_primary->new_tablespace('dropme_ts1'); +my $dropme_ts_primary2 = $node_primary->new_tablespace('dropme_ts2'); +my $soruce_ts_primary = $node_primary->new_tablespace('source_ts'); +my $target_ts_primary = $node_primary->new_tablespace('target_ts'); + +$node_primary->psql('postgres', +qq[ + CREATE TABLESPACE dropme_ts1 LOCATION '$dropme_ts_primary1'; + CREATE TABLESPACE dropme_ts2 LOCATION '$dropme_ts_primary2'; + CREATE TABLESPACE source_ts LOCATION '$soruce_ts_primary'; + CREATE TABLESPACE target_ts LOCATION '$target_ts_primary'; + CREATE DATABASE template_db IS_TEMPLATE = true; +]); +my $backup_name = 'my_backup'; +$node_primary->backup($backup_name); + +my $node_standby = get_new_node('standby2'); +$node_standby->init_from_backup($node_primary, $backup_name, has_streaming => 1); +$node_standby->start; + +# Make sure connection is made +$node_primary->poll_query_until( + 'postgres', 'SELECT count(*) = 1 FROM pg_stat_replication'); + +$node_standby->safe_psql('postgres', 'CHECKPOINT'); + +# Do immediate shutdown just after a sequence of CREAT DATABASE / DROP +# DATABASE / DROP TABLESPACE. This causes CREATE DATABASE WAL records +# to be applied to already-removed directories. +$node_primary->safe_psql('postgres', + q[CREATE DATABASE dropme_db1 WITH TABLESPACE dropme_ts1; + CREATE DATABASE dropme_db2 WITH TABLESPACE dropme_ts2; + CREATE DATABASE moveme_db TABLESPACE source_ts; + ALTER DATABASE moveme_db SET TABLESPACE target_ts; + CREATE DATABASE newdb TEMPLATE template_db; + ALTER DATABASE template_db IS_TEMPLATE = false; + DROP DATABASE dropme_db1; + DROP DATABASE dropme_db2; DROP TABLESPACE dropme_ts2; + DROP TABLESPACE source_ts; + DROP DATABASE template_db;]); + +$node_primary->wait_for_catchup($node_standby, 'replay', + $node_primary->lsn('replay')); +$node_standby->stop('immediate'); + +# Should restart ignoring directory creation error. +is($node_standby->start(fail_ok => 1), 1); + + +# TEST 5 +# +# Ensure that a missing tablespace directory during create database +# replay immediately causes panic if the standby has already reached +# consistent state (archive recovery is in progress). + +$node_primary = get_new_node('primary3'); +$node_primary->init(allows_streaming => 1); +$node_primary->start; + +# Create tablespace +my $ts_primary = $node_primary->new_tablespace('dropme_ts1'); +$node_primary->safe_psql('postgres', + "CREATE TABLESPACE ts1 LOCATION '$ts_primary'"); +$node_primary->safe_psql('postgres', "CREATE DATABASE db1 TABLESPACE ts1"); + +# Take backup +$backup_name = 'my_backup'; +$node_primary->backup($backup_name); +$node_standby = get_new_node('standby3'); +$node_standby->init_from_backup($node_primary, $backup_name, has_streaming => 1); +$node_standby->start; + +# Make sure standby reached consistency and starts accepting connections +$node_standby->poll_query_until('postgres', 'SELECT 1', '1'); + +# Remove standby tablespace directory so it will be missing when +# replay resumes. +File::Path::rmtree($node_standby->tablespace_dir('dropme_ts1')); + +# Create a database in the tablespace and a table in default tablespace +$node_primary->safe_psql('postgres', + q[CREATE TABLE should_not_replay_insertion(a int); + CREATE DATABASE db2 WITH TABLESPACE ts1; + INSERT INTO should_not_replay_insertion VALUES (1);]); + +# Standby should fail and should not silently skip replaying the wal +if ($node_primary->poll_query_until( + 'postgres', + 'SELECT count(*) = 0 FROM pg_stat_replication', + 't') == 1) +{ + pass('standby failed as expected'); + # We know that the standby has failed. Setting its pid to + # undefined avoids error when PostgreNode module tries to stop the + # standby node as part of tear_down sequence. + $node_standby->{_pid} = undef; +} +else +{ + fail('standby did not fail within 5 seconds'); +} + -- 2.27.0