From 33fd10bbff8105f612ea4e95e82ae6185827a583 Mon Sep 17 00:00:00 2001
From: "kuroda.hayato%40jp.fujitsu.com" <kuroda.hayato@jp.fujitsu.com>
Date: Mon, 27 Jun 2022 04:21:02 +0000
Subject: [PATCH v4 3/3] Tests for Logical Replication Group

This patch contains tap tests for Logical Replication Group(LRG).
---
 src/test/Makefile                      |   2 +-
 src/test/lrg/.gitignore                |   2 +
 src/test/lrg/Makefile                  |  23 +++
 src/test/lrg/t/001_validation.pl       | 226 +++++++++++++++++++++++++
 src/test/lrg/t/002_constructions.pl    | 137 +++++++++++++++
 src/test/lrg/t/003_rep.pl              | 205 ++++++++++++++++++++++
 src/test/regress/expected/oidjoins.out |   6 +
 7 files changed, 600 insertions(+), 1 deletion(-)
 create mode 100644 src/test/lrg/.gitignore
 create mode 100644 src/test/lrg/Makefile
 create mode 100644 src/test/lrg/t/001_validation.pl
 create mode 100644 src/test/lrg/t/002_constructions.pl
 create mode 100644 src/test/lrg/t/003_rep.pl

diff --git a/src/test/Makefile b/src/test/Makefile
index 69ef074d75..f12a8cbedf 100644
--- a/src/test/Makefile
+++ b/src/test/Makefile
@@ -12,7 +12,7 @@ subdir = src/test
 top_builddir = ../..
 include $(top_builddir)/src/Makefile.global
 
-SUBDIRS = perl regress isolation modules authentication recovery subscription
+SUBDIRS = perl regress isolation modules authentication recovery subscription lrg
 
 ifeq ($(with_icu),yes)
 SUBDIRS += icu
diff --git a/src/test/lrg/.gitignore b/src/test/lrg/.gitignore
new file mode 100644
index 0000000000..e07b677a7d
--- /dev/null
+++ b/src/test/lrg/.gitignore
@@ -0,0 +1,2 @@
+# Generated by regression tests
+/tmp_check/
diff --git a/src/test/lrg/Makefile b/src/test/lrg/Makefile
new file mode 100644
index 0000000000..065cabd6eb
--- /dev/null
+++ b/src/test/lrg/Makefile
@@ -0,0 +1,23 @@
+#-------------------------------------------------------------------------
+#
+# Makefile for src/test/lrg
+#
+# Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
+# Portions Copyright (c) 1994, Regents of the University of California
+#
+# src/test/lrg/Makefile
+#
+#-------------------------------------------------------------------------
+
+subdir = src/test/lrg
+top_builddir = ../../..
+include $(top_builddir)/src/Makefile.global
+
+check:
+	$(prove_check)
+
+installcheck:
+	$(prove_installcheck)
+
+clean distclean maintainer-clean:
+	rm -rf tmp_check
diff --git a/src/test/lrg/t/001_validation.pl b/src/test/lrg/t/001_validation.pl
new file mode 100644
index 0000000000..bf942cd78b
--- /dev/null
+++ b/src/test/lrg/t/001_validation.pl
@@ -0,0 +1,226 @@
+
+# Copyright (c) 2022, PostgreSQL Global Development Group
+
+#
+# Tests for constructing a logical replication groups
+#
+
+# Basic logical replication test
+use strict;
+use warnings;
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+my $result;
+my $stdout;
+my $stderr;
+
+#
+# In this test three nodes are used, called node1, node2, and node3.
+# After checking validations of APIs, N-way logical replication
+# group is created with following cases:
+#
+# - Case1: There is no data in any of the nodes
+#
+
+#
+# Initialize all nodes
+#
+
+my $node1 = PostgreSQL::Test::Cluster->new('node1');
+$node1->init(allows_streaming => 'logical');
+$node1->start;
+
+my $node2 = PostgreSQL::Test::Cluster->new('node2');
+$node2->init(allows_streaming => 'logical');
+$node2->start;
+
+my $node3 = PostgreSQL::Test::Cluster->new('node3');
+$node3->init(allows_streaming => 'logical');
+$node3->start;
+
+#
+# Input check for APIs
+#
+
+my $group_name = 'test_group';
+
+my $node1_connstr = $node1->connstr . ' dbname=postgres';
+my $node2_connstr = $node2->connstr . ' dbname=postgres';
+my $node3_connstr = $node3->connstr . ' dbname=postgres';
+
+##
+## Tests for lrg_create
+##
+
+$result = $node1->safe_psql('postgres', "SELECT COUNT(*) FROM pg_lrg_info");
+is($result, qq(0), 'check the initialized node does not belong to any group');
+
+($result, $stdout, $stderr) = $node1->psql(
+	'postgres', "SELECT lrg_create('$group_name', 'WRONG INPUT', '$node1_connstr', 'node1')");
+like
+(
+	$stderr,
+	qr/ERROR:  cannot create a node group
+DETAIL:  Only 'FOR ALL TABLES' is supported as publication type./,
+	"check only one publication type is allowed"
+);
+
+$node1->safe_psql(
+	'postgres',
+	"SELECT lrg_create('$group_name', 'FOR ALL TABLES', '$node1_connstr', 'node1')");
+
+$node1->safe_psql('postgres', "SELECT lrg_wait()");
+
+($result, $stdout, $stderr) = $node1->psql(
+	'postgres', "SELECT lrg_create('$group_name', 'FOR ALL TABLES', '$node1_connstr', 'node1')");
+like
+(
+	$stderr,
+	qr/ERROR:  could not create a node group
+DETAIL:  This node was already a member of $group_name.
+HINT:  You need to detach from or drop the group./,
+	"check duplicated calling of lrg_create is not allowed"
+);
+
+##
+## Tests for lrg_node_attach
+##
+
+($result, $stdout, $stderr) = $node2->psql(
+	'postgres', "SELECT lrg_node_attach('WRONG GROUP', '$node2_connstr', '$node1_connstr', 'node2')");
+like
+(
+	$stderr,
+	qr/ERROR:  could not attach to the node group
+DETAIL:  Upstream node is not a member of specified group./,
+	"sanity check for group_name while attaching"
+);
+
+($result, $stdout, $stderr) = $node2->psql(
+	'postgres', "SELECT lrg_node_attach('$group_name', '$node2_connstr', 'wrong connection string', 'node2')");
+like
+(
+	$stderr,
+	qr/ERROR:  could not connect to the server
+HINT:  Please check the connection string and health of destination./,
+	"sanity check for upstream connection string"
+);
+
+$node2->safe_psql(
+	'postgres',
+	"SELECT lrg_node_attach('$group_name', '$node2_connstr', '$node1_connstr', 'node2')");
+
+$node2->safe_psql('postgres', "SELECT lrg_wait()");
+
+($result, $stdout, $stderr) = $node2->psql(
+	'postgres', "SELECT lrg_node_attach('$group_name', '$node2_connstr', 'wrong connection string', 'node2')");
+like
+(
+	$stderr,
+	qr/ERROR:  could not attach to a node group
+DETAIL:  This node was already a member of $group_name.
+HINT:  You need to detach from or drop the group./,
+	"check duplicated calling of lrg_node_attach is not allowed"
+);
+
+($result, $stdout, $stderr) = $node2->psql(
+	'postgres', "SELECT lrg_create('$group_name', 'FOR ALL TABLES', '$node1_connstr', 'node1')");
+like
+(
+	$stderr,
+	qr/ERROR:  could not create a node group
+DETAIL:  This node was already a member of $group_name.
+HINT:  You need to detach from or drop the group./,
+	"check lrg_create cannot be called if this node is already a member of a node group"
+);
+
+###
+### do lrg_node_attach() in node3. It will be used for testing "force" detach.
+###
+
+$node3->safe_psql(
+	'postgres',
+	"SELECT lrg_node_attach('$group_name', '$node3_connstr', '$node1_connstr', 'node3')");
+
+$node3->safe_psql('postgres', "SELECT lrg_wait()");
+
+
+##
+## Tests for lrg_node_detach
+##
+
+($result, $stdout, $stderr) = $node2->psql(
+	'postgres', "SELECT lrg_node_detach('WRONG GROUP', 'node2')");
+like
+(
+	$stderr,
+	qr/ERROR:  could not detach from the node group
+DETAIL:  This node was in $group_name, but WRONG GROUP is specified./,
+	"sanity check for group_name while detaching"
+);
+
+$node2->safe_psql(
+	'postgres',
+	"SELECT lrg_node_detach('$group_name', 'node2')");
+
+$node2->safe_psql('postgres', "SELECT lrg_wait()");
+
+($result, $stdout, $stderr) = $node2->psql(
+	'postgres', "SELECT lrg_node_detach('$group_name', 'node2')");
+like
+(
+	$stderr,
+	qr/ERROR:  could not detach from the node group
+DETAIL:  This node was in any node groups./,
+	"duplicated calling is not allowed"
+);
+
+###
+### Tests for "force" detach. It can be done even if specified node goes down
+###
+
+$node3->stop('fast');
+
+$node1->safe_psql(
+	'postgres',
+	"SELECT lrg_node_detach('$group_name', 'node3', true)");
+
+$node1->safe_psql('postgres', "SELECT lrg_wait()");
+
+##
+## Tests for lrg_drop
+##
+
+($result, $stdout, $stderr) = $node1->psql(
+	'postgres', "SELECT lrg_drop('WRONG GROUP')");
+like
+(
+	$stderr,
+	qr/ERROR:  could not drop the node group
+DETAIL:  This node was in $group_name, but WRONG GROUP is specified./,
+	"sanity check for group_name while dropping"
+);
+
+$node1->safe_psql(
+	'postgres',
+	"SELECT lrg_drop('$group_name')");
+
+$node1->safe_psql('postgres', "SELECT lrg_wait()");
+
+($result, $stdout, $stderr) = $node1->psql(
+	'postgres', "SELECT lrg_drop('$group_name')");
+like
+(
+	$stderr,
+	qr/ERROR:  could not drop the node group
+DETAIL:  This node was in any node groups./,
+	"duplicated calling is not allowed"
+);
+
+# shutdown
+$node1->stop('fast');
+$node2->stop('fast');
+
+done_testing();
diff --git a/src/test/lrg/t/002_constructions.pl b/src/test/lrg/t/002_constructions.pl
new file mode 100644
index 0000000000..b63eb5bfa5
--- /dev/null
+++ b/src/test/lrg/t/002_constructions.pl
@@ -0,0 +1,137 @@
+# Copyright (c) 2022, PostgreSQL Global Development Group
+
+#
+# Tests for constructing a logical replication groups
+#
+
+# Basic logical replication test
+use strict;
+use warnings;
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+my $result;
+my $stdout;
+my $stderr;
+
+#
+# In this test two-way logical replication group
+# will be created. A node will attach, detach, and attach
+# again to the same node group. Also, catalogs related with
+# LRG will be checked on both nodes
+#
+
+#
+# Initialize all nodes
+#
+
+my $node1 = PostgreSQL::Test::Cluster->new('node1');
+$node1->init(allows_streaming => 'logical');
+$node1->start;
+
+my $node2 = PostgreSQL::Test::Cluster->new('node2');
+$node2->init(allows_streaming => 'logical');
+$node2->start;
+
+my $group_name = 'test_group';
+
+my $node1_connstr = $node1->connstr . ' dbname=postgres';
+my $node2_connstr = $node2->connstr . ' dbname=postgres';
+
+# Create a logical replication group on node1
+
+$node1->safe_psql(
+	'postgres',
+	"SELECT lrg_create('$group_name', 'FOR ALL TABLES', '$node1_connstr', 'node1')");
+
+## Check its catalog related with LRG
+
+$result = $node1->safe_psql('postgres', "SELECT groupname, pub_type FROM pg_lrg_info");
+is($result, qq($group_name|FOR ALL TABLES), 'check creating group has been succeeded');
+
+$result = $node1->safe_psql('postgres', "SELECT nodename FROM pg_lrg_nodes");
+is($result, qq(node1), 'check this node has attached to the group');
+
+$node1->safe_psql('postgres', "SELECT lrg_wait()");
+
+$result = $node1->safe_psql('postgres', "SELECT COUNT(*) FROM pg_lrg_pub");
+is($result, qq(1), 'check a publication has been associated with the group');
+
+# Attach node2 to the logical replication group
+
+$node2->safe_psql(
+	'postgres',
+	"SELECT lrg_node_attach('$group_name', '$node2_connstr', '$node1_connstr', 'node2')");
+
+$node2->safe_psql('postgres', "SELECT lrg_wait()");
+
+## Check its catalog related with LRG
+
+$result = $node2->safe_psql('postgres', "SELECT groupname, pub_type FROM pg_lrg_info");
+is($result, qq($group_name|FOR ALL TABLES), 'check creating group has been succeeded');
+
+$result = $node2->safe_psql('postgres', "SELECT COUNT(*) FROM pg_lrg_nodes");
+is($result, qq(2), 'check this node has attached to the group');
+
+$result = $node2->safe_psql('postgres', "SELECT COUNT(*) FROM pg_lrg_pub");
+is($result, qq(1), 'check a publication has been associated with the group');
+
+$result = $node1->safe_psql('postgres', "SELECT COUNT(*) FROM pg_lrg_sub");
+is($result, qq(1), 'check a subscription has been associated with the group');
+
+$result = $node2->safe_psql('postgres', "SELECT COUNT(*) FROM pg_lrg_sub");
+is($result, qq(1), 'check a subscription has been associated with the group');
+
+# Detach node2 once
+
+$node2->safe_psql('postgres', "SELECT lrg_node_detach('$group_name', 'node2')");
+$node2->safe_psql('postgres', "SELECT lrg_wait()");
+
+## Check its catalog related with LRG
+
+$result = $node2->safe_psql('postgres', "SELECT groupname, pub_type FROM pg_lrg_info");
+is($result, qq(), 'check catalogs have been cleaned up');
+
+$result = $node2->safe_psql('postgres', "SELECT COUNT(*) FROM pg_lrg_nodes");
+is($result, qq(0), 'check catalogs have been cleaned up');
+
+$result = $node2->safe_psql('postgres', "SELECT COUNT(*) FROM pg_lrg_pub");
+is($result, qq(0), 'check catalogs have been cleaned up');
+
+$result = $node1->safe_psql('postgres', "SELECT COUNT(*) FROM pg_lrg_sub");
+is($result, qq(1), 'check a subscription has been deleted');
+
+$result = $node2->safe_psql('postgres', "SELECT COUNT(*) FROM pg_lrg_sub");
+is($result, qq(0), 'check catalogs have been cleaned up');
+
+# ...and attach again
+
+$node2->safe_psql(
+	'postgres',
+	"SELECT lrg_node_attach('$group_name', '$node2_connstr', '$node1_connstr', 'node2')");
+
+$node2->safe_psql('postgres', "SELECT lrg_wait()");
+
+## Check its catalog related with LRG
+
+$result = $node2->safe_psql('postgres', "SELECT groupname, pub_type FROM pg_lrg_info");
+is($result, qq($group_name|FOR ALL TABLES), 'check creating group has been succeeded');
+
+$result = $node2->safe_psql('postgres', "SELECT COUNT(*) FROM pg_lrg_nodes");
+is($result, qq(2), 'check this node has attached to the group');
+
+$result = $node2->safe_psql('postgres', "SELECT COUNT(*) FROM pg_lrg_pub");
+is($result, qq(1), 'check a publication has been associated with the group');
+
+$result = $node1->safe_psql('postgres', "SELECT COUNT(*) FROM pg_lrg_sub");
+is($result, qq(2), 'check a subscription has been associated with the group');
+
+$result = $node2->safe_psql('postgres', "SELECT COUNT(*) FROM pg_lrg_sub");
+is($result, qq(1), 'check a subscription has been associated with the group');
+
+# shutdown
+$node1->stop('fast');
+$node2->stop('fast');
+
+done_testing();
diff --git a/src/test/lrg/t/003_rep.pl b/src/test/lrg/t/003_rep.pl
new file mode 100644
index 0000000000..82e0da475a
--- /dev/null
+++ b/src/test/lrg/t/003_rep.pl
@@ -0,0 +1,205 @@
+# Copyright (c) 2022, PostgreSQL Global Development Group
+
+#
+# Basic LRG tests: checks replications of changes on nodes
+#
+
+# Basic logical replication test
+use strict;
+use warnings;
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+my $result;
+my $stdout;
+my $stderr;
+
+#
+# In this test three-way logical replication group
+# will be created with following cases:
+#
+# - Case1: There is no data in any of the nodes
+#
+
+#
+# Initialize all nodes
+#
+
+my $node1 = PostgreSQL::Test::Cluster->new('node1');
+$node1->init(allows_streaming => 'logical');
+$node1->start;
+
+my $node2 = PostgreSQL::Test::Cluster->new('node2');
+$node2->init(allows_streaming => 'logical');
+$node2->start;
+
+my $node3 = PostgreSQL::Test::Cluster->new('node3');
+$node3->init(allows_streaming => 'logical');
+$node3->start;
+
+my $group_name = 'test_group';
+
+my $node1_connstr = $node1->connstr . ' dbname=postgres';
+my $node2_connstr = $node2->connstr . ' dbname=postgres';
+my $node3_connstr = $node3->connstr . ' dbname=postgres';
+
+#
+# Create tables that will share changes on all nodes
+# They will be used for testing
+#
+
+$node1->safe_psql(
+	'postgres',
+	'CREATE TABLE foo (id int PRIMARY KEY)');
+$node2->safe_psql(
+	'postgres',
+	'CREATE TABLE foo (id int PRIMARY KEY)');
+$node3->safe_psql(
+	'postgres',
+	'CREATE TABLE foo (id int PRIMARY KEY)');
+
+#
+# - Case1: There is no data in any of the nodes
+#
+
+## Create a logical replication group on node1
+
+$node1->safe_psql(
+	'postgres',
+	"SELECT lrg_create('$group_name', 'FOR ALL TABLES', '$node1_connstr', 'node1')");
+
+## Attach node2, node3 to the logical replication group
+
+$node2->safe_psql(
+	'postgres',
+	"SELECT lrg_node_attach('$group_name', '$node2_connstr', '$node1_connstr', 'node2')");
+
+$node2->safe_psql('postgres', "SELECT lrg_wait()");
+
+$node3->safe_psql(
+	'postgres',
+	"SELECT lrg_node_attach('$group_name', '$node3_connstr', '$node1_connstr', 'node3')");
+
+$node3->safe_psql('postgres', "SELECT lrg_wait()");
+
+## Now logical replication system has been created. All changes will be shared to all nodes.
+
+my $subname12 = $node1->safe_psql('postgres',
+	"SELECT subname FROM pg_subscription WHERE subconninfo = '$node2_connstr'");
+my $subname13 = $node1->safe_psql('postgres',
+	"SELECT subname FROM pg_subscription WHERE subconninfo = '$node3_connstr'");
+
+my $subname21 = $node2->safe_psql('postgres',
+	"SELECT subname FROM pg_subscription WHERE subconninfo = '$node1_connstr'");
+my $subname23 = $node2->safe_psql('postgres',
+	"SELECT subname FROM pg_subscription WHERE subconninfo = '$node3_connstr'");
+
+my $subname31 = $node3->safe_psql('postgres',
+	"SELECT subname FROM pg_subscription WHERE subconninfo = '$node1_connstr'");
+my $subname32 = $node3->safe_psql('postgres',
+	"SELECT subname FROM pg_subscription WHERE subconninfo = '$node2_connstr'");
+
+## Inserted data on node1 will appear on node2, node3.
+
+$node1->safe_psql('postgres', "INSERT INTO foo VALUES (1)");
+
+verify_data($node1, $node2, $node3, '1');
+
+## Inserted data on node2 will appear on node1, node3.
+
+$node2->safe_psql('postgres', "INSERT INTO foo VALUES (2)");
+
+verify_data($node1, $node2, $node3, '1
+2');
+
+## Inserted data on node3 will appear on node1, node2.
+
+$node3->safe_psql('postgres', "INSERT INTO foo VALUES (3)");
+
+verify_data($node1, $node2, $node3, '1
+2
+3');
+
+
+## Updated data on node1 will appear on node2, node3.
+
+$node1->safe_psql('postgres', "UPDATE foo SET id = 4 WHERE id = 1");
+
+verify_data($node1, $node2, $node3, '2
+3
+4');
+
+## Updated data on node2 will appear on node1, node3.
+
+$node2->safe_psql('postgres', "UPDATE foo SET id = 5 WHERE id = 2");
+
+verify_data($node1, $node2, $node3, '3
+4
+5');
+
+## Updated data on node3 will appear on node1, node2.
+
+$node2->safe_psql('postgres', "UPDATE foo SET id = 6 WHERE id = 3");
+
+verify_data($node1, $node2, $node3, '4
+5
+6');
+
+## Deleted data on node1 will be removed on node2, node3.
+
+$node1->safe_psql('postgres', "DELETE FROM foo WHERE id = 6");
+
+verify_data($node1, $node2, $node3, '4
+5');
+
+## Deleted data on node2 will be removed on node1, node3.
+
+$node2->safe_psql('postgres', "DELETE FROM foo WHERE id = 5");
+
+verify_data($node1, $node2, $node3, '4');
+
+## Deleted data on node3 will be removed on node1, node2.
+
+$node2->safe_psql('postgres', "DELETE FROM foo WHERE id = 4");
+
+verify_data($node1, $node2, $node3, '');
+
+
+# shutdown
+$node1->stop('fast');
+$node2->stop('fast');
+
+done_testing();
+
+# Subroutine to verify the data is replicated successfully.
+sub verify_data
+{
+	my ($node1, $node2, $node3, $expect) = @_;
+
+	$node1->wait_for_catchup($subname21);
+	$node1->wait_for_catchup($subname31);
+	$node2->wait_for_catchup($subname12);
+	$node2->wait_for_catchup($subname32);
+	$node3->wait_for_catchup($subname13);
+	$node3->wait_for_catchup($subname23);
+
+	# check that data is replicated to all the nodes
+	$result =
+	  $node1->safe_psql('postgres', "SELECT * FROM foo ORDER BY 1;");
+	is($result, qq($expect),
+	   'Data is replicated as expected'
+	);
+
+	$result =
+	  $node2->safe_psql('postgres', "SELECT * FROM foo ORDER BY 1;");
+	is($result, qq($expect),
+	   'Data is replicated as expected'
+	);
+
+	$result =
+	  $node3->safe_psql('postgres', "SELECT * FROM foo ORDER BY 1;");
+	is($result, qq($expect),
+	   'Data is replicated as expected'
+	);
+}
diff --git a/src/test/regress/expected/oidjoins.out b/src/test/regress/expected/oidjoins.out
index 215eb899be..da0ef150a2 100644
--- a/src/test/regress/expected/oidjoins.out
+++ b/src/test/regress/expected/oidjoins.out
@@ -266,3 +266,9 @@ NOTICE:  checking pg_subscription {subdbid} => pg_database {oid}
 NOTICE:  checking pg_subscription {subowner} => pg_authid {oid}
 NOTICE:  checking pg_subscription_rel {srsubid} => pg_subscription {oid}
 NOTICE:  checking pg_subscription_rel {srrelid} => pg_class {oid}
+NOTICE:  checking pg_lrg_nodes {groupid} => pg_lrg_info {oid}
+NOTICE:  checking pg_lrg_nodes {dbid} => pg_database {oid}
+NOTICE:  checking pg_lrg_pub {groupid} => pg_lrg_info {oid}
+NOTICE:  checking pg_lrg_pub {pubid} => pg_publication {oid}
+NOTICE:  checking pg_lrg_sub {groupid} => pg_lrg_info {oid}
+NOTICE:  checking pg_lrg_sub {subid} => pg_subscription {oid}
-- 
2.27.0

