From 2d84095db2a68290e0e462ed7e84759eeed3568f Mon Sep 17 00:00:00 2001 From: Antonin Houska Date: Wed, 26 Feb 2025 09:17:20 +0100 Subject: [PATCH 6/9] Add regression tests. As this patch series adds the CONCURRENTLY option to the REPACK command, it's appropriate to test that the "concurrent data changes" (i.e. changes done by application while we are copying the table contents to the new storage) are processed correctly. Injection points are used to stop the data copying at some point. While the backend in charge of the copying is waiting on the injection point, another backend runs some INSERT, UPDATE and DELETE commands on the table. Then we wake up the first backend and let the REPACK CONCURRENTLY command finish. Finally we check that all the "concurrent data changes" are present in the table and that they contain the correct visibility information. --- src/backend/commands/cluster.c | 7 + src/test/modules/injection_points/Makefile | 3 +- .../injection_points/expected/repack.out | 113 ++++++++++++++ .../modules/injection_points/logical.conf | 1 + src/test/modules/injection_points/meson.build | 4 + .../injection_points/specs/repack.spec | 140 ++++++++++++++++++ 6 files changed, 267 insertions(+), 1 deletion(-) create mode 100644 src/test/modules/injection_points/expected/repack.out create mode 100644 src/test/modules/injection_points/logical.conf create mode 100644 src/test/modules/injection_points/specs/repack.spec diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index b336d760f2..1ff0fcd1d9 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -59,6 +59,7 @@ #include "utils/formatting.h" #include "utils/fmgroids.h" #include "utils/guc.h" +#include "utils/injection_point.h" #include "utils/inval.h" #include "utils/lsyscache.h" #include "utils/memutils.h" @@ -3710,6 +3711,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ident_key = build_identity_key(ident_idx_new, OldHeap, &ident_key_nentries); + /* + * During testing, wait for another backend to perform concurrent data + * changes which we will process below. + */ + INJECTION_POINT("repack-concurrently-before-lock"); + /* * Flush all WAL records inserted so far (possibly except for the last * incomplete page, see GetInsertRecPtr), to minimize the amount of data diff --git a/src/test/modules/injection_points/Makefile b/src/test/modules/injection_points/Makefile index e680991f8d..405d0811b4 100644 --- a/src/test/modules/injection_points/Makefile +++ b/src/test/modules/injection_points/Makefile @@ -14,7 +14,8 @@ PGFILEDESC = "injection_points - facility for injection points" REGRESS = injection_points hashagg reindex_conc REGRESS_OPTS = --dlpath=$(top_builddir)/src/test/regress -ISOLATION = basic inplace syscache-update-pruned +ISOLATION = basic inplace syscache-update-pruned repack +ISOLATION_OPTS = --temp-config $(top_srcdir)/src/test/modules/injection_points/logical.conf TAP_TESTS = 1 diff --git a/src/test/modules/injection_points/expected/repack.out b/src/test/modules/injection_points/expected/repack.out new file mode 100644 index 0000000000..49a736ed61 --- /dev/null +++ b/src/test/modules/injection_points/expected/repack.out @@ -0,0 +1,113 @@ +Parsed test spec with 2 sessions + +starting permutation: wait_before_lock change_existing change_new change_subxact1 change_subxact2 check2 wakeup_before_lock check1 +injection_points_attach +----------------------- + +(1 row) + +step wait_before_lock: + REPACK CONCURRENTLY repack_test USING INDEX repack_test_pkey; + +step change_existing: + UPDATE repack_test SET i=10 where i=1; + UPDATE repack_test SET j=20 where i=2; + UPDATE repack_test SET i=30 where i=3; + UPDATE repack_test SET i=40 where i=30; + DELETE FROM repack_test WHERE i=4; + +step change_new: + INSERT INTO repack_test(i, j) VALUES (5, 5), (6, 6), (7, 7), (8, 8); + UPDATE repack_test SET i=50 where i=5; + UPDATE repack_test SET j=60 where i=6; + DELETE FROM repack_test WHERE i=7; + +step change_subxact1: + BEGIN; + INSERT INTO repack_test(i, j) VALUES (100, 100); + SAVEPOINT s1; + UPDATE repack_test SET i=101 where i=100; + SAVEPOINT s2; + UPDATE repack_test SET i=102 where i=101; + COMMIT; + +step change_subxact2: + BEGIN; + SAVEPOINT s1; + INSERT INTO repack_test(i, j) VALUES (110, 110); + ROLLBACK TO SAVEPOINT s1; + INSERT INTO repack_test(i, j) VALUES (110, 111); + COMMIT; + +step check2: + INSERT INTO relfilenodes(node) + SELECT relfilenode FROM pg_class WHERE relname='repack_test'; + + SELECT i, j FROM repack_test ORDER BY i, j; + + INSERT INTO data_s2(_xmin, _cmin, i, j) + SELECT xmin, cmin, i, j FROM repack_test; + + i| j +---+--- + 2| 20 + 6| 60 + 8| 8 + 10| 1 + 40| 3 + 50| 5 +102|100 +110|111 +(8 rows) + +step wakeup_before_lock: + SELECT injection_points_wakeup('repack-concurrently-before-lock'); + +injection_points_wakeup +----------------------- + +(1 row) + +step wait_before_lock: <... completed> +step check1: + INSERT INTO relfilenodes(node) + SELECT relfilenode FROM pg_class WHERE relname='repack_test'; + + SELECT count(DISTINCT node) FROM relfilenodes; + + SELECT i, j FROM repack_test ORDER BY i, j; + + INSERT INTO data_s1(_xmin, _cmin, i, j) + SELECT xmin, cmin, i, j FROM repack_test; + + SELECT count(*) + FROM data_s1 d1 FULL JOIN data_s2 d2 USING (_xmin, _cmin, i, j) + WHERE d1.i ISNULL OR d2.i ISNULL; + +count +----- + 2 +(1 row) + + i| j +---+--- + 2| 20 + 6| 60 + 8| 8 + 10| 1 + 40| 3 + 50| 5 +102|100 +110|111 +(8 rows) + +count +----- + 0 +(1 row) + +injection_points_detach +----------------------- + +(1 row) + diff --git a/src/test/modules/injection_points/logical.conf b/src/test/modules/injection_points/logical.conf new file mode 100644 index 0000000000..c8f264bc6c --- /dev/null +++ b/src/test/modules/injection_points/logical.conf @@ -0,0 +1 @@ +wal_level = logical \ No newline at end of file diff --git a/src/test/modules/injection_points/meson.build b/src/test/modules/injection_points/meson.build index d61149712f..0e3c47ba99 100644 --- a/src/test/modules/injection_points/meson.build +++ b/src/test/modules/injection_points/meson.build @@ -46,9 +46,13 @@ tests += { 'specs': [ 'basic', 'inplace', + 'repack', 'syscache-update-pruned', ], 'runningcheck': false, # see syscache-update-pruned + # 'repack' requires wal_level = 'logical'. + 'regress_args': ['--temp-config', files('logical.conf')], + }, 'tap': { 'env': { diff --git a/src/test/modules/injection_points/specs/repack.spec b/src/test/modules/injection_points/specs/repack.spec new file mode 100644 index 0000000000..5aa8983f98 --- /dev/null +++ b/src/test/modules/injection_points/specs/repack.spec @@ -0,0 +1,140 @@ +# Prefix the system columns with underscore as they are not allowed as column +# names. +setup +{ + CREATE EXTENSION injection_points; + + CREATE TABLE repack_test(i int PRIMARY KEY, j int); + INSERT INTO repack_test(i, j) VALUES (1, 1), (2, 2), (3, 3), (4, 4); + + CREATE TABLE relfilenodes(node oid); + + CREATE TABLE data_s1(_xmin xid, _cmin cid, i int, j int); + CREATE TABLE data_s2(_xmin xid, _cmin cid, i int, j int); +} + +teardown +{ + DROP TABLE repack_test; + DROP EXTENSION injection_points; + + DROP TABLE relfilenodes; + DROP TABLE data_s1; + DROP TABLE data_s2; +} + +session s1 +setup +{ + SELECT injection_points_set_local(); + SELECT injection_points_attach('repack-concurrently-before-lock', 'wait'); +} +# Perform the initial load and wait for s2 to do some data changes. +step wait_before_lock +{ + REPACK CONCURRENTLY repack_test USING INDEX repack_test_pkey; +} +# Check the table from the perspective of s1. +# +# Besides the contents, we also check that relfilenode has changed. +# +# xmin and cmin columns are used to check that we do not change tuple +# visibility information. Since we do not expect xmin to stay unchanged across +# test runs, it cannot appear in the output text. Instead, have each session +# write the contents into a table and use FULL JOIN to check if the outputs +# are identical. +step check1 +{ + INSERT INTO relfilenodes(node) + SELECT relfilenode FROM pg_class WHERE relname='repack_test'; + + SELECT count(DISTINCT node) FROM relfilenodes; + + SELECT i, j FROM repack_test ORDER BY i, j; + + INSERT INTO data_s1(_xmin, _cmin, i, j) + SELECT xmin, cmin, i, j FROM repack_test; + + SELECT count(*) + FROM data_s1 d1 FULL JOIN data_s2 d2 USING (_xmin, _cmin, i, j) + WHERE d1.i ISNULL OR d2.i ISNULL; +} +teardown +{ + SELECT injection_points_detach('repack-concurrently-before-lock'); +} + +session s2 +# Change the existing data. UPDATE changes both key and non-key columns. Also +# update one row twice to test whether tuple version generated by this session +# can be found. +step change_existing +{ + UPDATE repack_test SET i=10 where i=1; + UPDATE repack_test SET j=20 where i=2; + UPDATE repack_test SET i=30 where i=3; + UPDATE repack_test SET i=40 where i=30; + DELETE FROM repack_test WHERE i=4; +} +# Insert new rows and UPDATE / DELETE some of them. Again, update both key and +# non-key column. +step change_new +{ + INSERT INTO repack_test(i, j) VALUES (5, 5), (6, 6), (7, 7), (8, 8); + UPDATE repack_test SET i=50 where i=5; + UPDATE repack_test SET j=60 where i=6; + DELETE FROM repack_test WHERE i=7; +} + +# When applying concurrent data changes, we should see the effects of an +# in-progress subtransaction. +step change_subxact1 +{ + BEGIN; + INSERT INTO repack_test(i, j) VALUES (100, 100); + SAVEPOINT s1; + UPDATE repack_test SET i=101 where i=100; + SAVEPOINT s2; + UPDATE repack_test SET i=102 where i=101; + COMMIT; +} + +# When applying concurrent data changes, we should not see the effects of a +# rolled back subtransaction. +step change_subxact2 +{ + BEGIN; + SAVEPOINT s1; + INSERT INTO repack_test(i, j) VALUES (110, 110); + ROLLBACK TO SAVEPOINT s1; + INSERT INTO repack_test(i, j) VALUES (110, 111); + COMMIT; +} + +# Check the table from the perspective of s2. +step check2 +{ + INSERT INTO relfilenodes(node) + SELECT relfilenode FROM pg_class WHERE relname='repack_test'; + + SELECT i, j FROM repack_test ORDER BY i, j; + + INSERT INTO data_s2(_xmin, _cmin, i, j) + SELECT xmin, cmin, i, j FROM repack_test; +} +step wakeup_before_lock +{ + SELECT injection_points_wakeup('repack-concurrently-before-lock'); +} + +# Test if data changes introduced while one session is performing REPACK +# CONCURRENTLY find their way into the table. +permutation + wait_before_lock + change_existing + change_new + change_subxact1 + change_subxact2 + check2 + wakeup_before_lock + check1 -- 2.43.5