From 1bc42a534f186ae021cb425e23c137803bcfb473 Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Tue, 10 Mar 2026 09:28:15 -0400 Subject: [PATCH v39 1/8] Improve test coverage for heap-only tuple (HOT) update behavior Add regression coverage for the existing Heap-Only Tuple (HOT) update rules, so that later changes to them show up as changes to the expected output. The new hot_updates test in contrib/pageinspect covers: - HOT and non-HOT updates, and the HOT chain an index scan walks - summarizing (BRIN) indexes, which do not block HOT - TOAST columns - partial, multi-column and unique indexes - partitioned tables - expression indexes whose input changes while the expression's value does or does not, including expressions stored as another type - partial indexes, with the row inside, outside, entering or leaving the predicate - BEFORE ROW triggers that change an indexed column Discussion: https://postgr.es/m/78574B24-BE0A-42C5-8075-3FA9FA63B8FC@amazon.com --- contrib/pageinspect/Makefile | 3 +- contrib/pageinspect/expected/hot_updates.out | 801 ++++++++++++++++++ contrib/pageinspect/meson.build | 1 + contrib/pageinspect/sql/hot_updates.sql | 562 ++++++++++++ .../isolation/expected/update-epq-indexes.out | 94 ++ src/test/isolation/isolation_schedule | 1 + .../isolation/specs/update-epq-indexes.spec | 55 ++ 7 files changed, 1516 insertions(+), 1 deletion(-) create mode 100644 contrib/pageinspect/expected/hot_updates.out create mode 100644 contrib/pageinspect/sql/hot_updates.sql create mode 100644 src/test/isolation/expected/update-epq-indexes.out create mode 100644 src/test/isolation/specs/update-epq-indexes.spec diff --git a/contrib/pageinspect/Makefile b/contrib/pageinspect/Makefile index eae989569d0..231bd82137e 100644 --- a/contrib/pageinspect/Makefile +++ b/contrib/pageinspect/Makefile @@ -32,7 +32,8 @@ REGRESS = \ gin \ gist \ hash \ - oldextversions + oldextversions \ + hot_updates ifdef USE_PGXS PG_CONFIG = pg_config diff --git a/contrib/pageinspect/expected/hot_updates.out b/contrib/pageinspect/expected/hot_updates.out new file mode 100644 index 00000000000..17240104daf --- /dev/null +++ b/contrib/pageinspect/expected/hot_updates.out @@ -0,0 +1,801 @@ +-- +-- HOT_UPDATES +-- Test Heap-Only Tuple (HOT) update decisions +-- +-- Expected output records which UPDATEs are HOT, so that a change to the +-- rules shows up as a change to this file. +-- +-- Validation methods: +-- 1. Statistics (pg_stat_get_tuples_hot_updated) +-- 2. pageinspect for HOT chain structure +-- 3. EXPLAIN to confirm the planner still picks the index +-- +-- Load required extensions +CREATE EXTENSION IF NOT EXISTS pageinspect; +-- Sum of committed and in-progress (non-HOT, HOT) update counters. +CREATE OR REPLACE FUNCTION get_hot_count(rel_name text) +RETURNS TABLE ( + updates BIGINT, + hot BIGINT +) AS $$ +DECLARE + rel_oid oid; +BEGIN + rel_oid := rel_name::regclass::oid; + updates := COALESCE(pg_stat_get_tuples_updated(rel_oid), 0) + + COALESCE(pg_stat_get_xact_tuples_updated(rel_oid), 0); + hot := COALESCE(pg_stat_get_tuples_hot_updated(rel_oid), 0) + + COALESCE(pg_stat_get_xact_tuples_hot_updated(rel_oid), 0); + RETURN NEXT; +END; +$$ LANGUAGE plpgsql; +-- True iff target_ctid is the TAIL of a HOT chain on the same page. +CREATE OR REPLACE FUNCTION has_hot_chain(rel_name text, target_ctid tid) +RETURNS boolean AS $$ +DECLARE + block_num int; + page_item record; +BEGIN + block_num := (target_ctid::text::point)[0]::int; + FOR page_item IN + SELECT lp, lp_flags, t_ctid + FROM heap_page_items(get_raw_page(rel_name, block_num)) + WHERE lp_flags = 1 + AND t_ctid IS NOT NULL + AND t_ctid = target_ctid + AND ('(' || block_num::text || ',' || lp::text || ')')::tid != target_ctid + LOOP + RETURN true; + END LOOP; + RETURN false; +END; +$$ LANGUAGE plpgsql; +-- Emit the HOT chain rooted at start_ctid. +CREATE OR REPLACE FUNCTION print_hot_chain(rel_name text, start_ctid tid) +RETURNS TABLE(chain_position int, ctid tid, lp_flags text, t_ctid tid, chain_end boolean) AS +$$ +#variable_conflict use_column +DECLARE + block_num int; + line_ptr int; + current_ctid tid := start_ctid; + next_ctid tid; + position int := 0; + max_iterations int := 100; + page_item record; + found_predecessor boolean := false; + flags_name text; +BEGIN + block_num := (start_ctid::text::point)[0]::int; + + FOR page_item IN + SELECT lp, lp_flags, t_ctid + FROM heap_page_items(get_raw_page(rel_name, block_num)) + WHERE lp_flags = 1 + AND t_ctid = start_ctid + LOOP + current_ctid := ('(' || block_num::text || ',' || page_item.lp::text || ')')::tid; + found_predecessor := true; + EXIT; + END LOOP; + IF NOT found_predecessor THEN + current_ctid := start_ctid; + END IF; + + WHILE position < max_iterations LOOP + line_ptr := (current_ctid::text::point)[1]::int; + FOR page_item IN + SELECT lp, lp_flags, t_ctid + FROM heap_page_items(get_raw_page(rel_name, block_num)) + WHERE lp = line_ptr + LOOP + flags_name := CASE page_item.lp_flags + WHEN 0 THEN 'unused (0)' + WHEN 1 THEN 'normal (1)' + WHEN 2 THEN 'redirect (2)' + WHEN 3 THEN 'dead (3)' + ELSE 'unknown (' || page_item.lp_flags::text || ')' + END; + RETURN QUERY SELECT + position, + current_ctid, + flags_name, + page_item.t_ctid, + (page_item.t_ctid IS NULL OR page_item.t_ctid = current_ctid)::boolean; + + IF page_item.t_ctid IS NULL OR page_item.t_ctid = current_ctid THEN + RETURN; + END IF; + next_ctid := page_item.t_ctid; + IF (next_ctid::text::point)[0]::int != block_num THEN + RETURN; + END IF; + current_ctid := next_ctid; + position := position + 1; + END LOOP; + IF position = 0 THEN + RETURN; + END IF; + END LOOP; +END; +$$ LANGUAGE plpgsql; +-- --------------------------------------------------------------------------- +-- Basic HOT: update of a non-indexed column +-- --------------------------------------------------------------------------- +CREATE TABLE hot_test ( + id int PRIMARY KEY, + indexed_col int, + non_indexed_col text +) WITH (fillfactor = 50); +CREATE INDEX hot_test_indexed_idx ON hot_test(indexed_col); +INSERT INTO hot_test VALUES (1, 100, 'initial'); +INSERT INTO hot_test VALUES (2, 200, 'initial'); +INSERT INTO hot_test VALUES (3, 300, 'initial'); +SELECT pg_stat_force_next_flush(); + pg_stat_force_next_flush +-------------------------- + +(1 row) + +SELECT * FROM get_hot_count('hot_test'); + updates | hot +---------+----- + 0 | 0 +(1 row) + +-- Three classic HOT updates (non-indexed col). +UPDATE hot_test SET non_indexed_col = 'updated1' WHERE id = 1; +UPDATE hot_test SET non_indexed_col = 'updated2' WHERE id = 2; +UPDATE hot_test SET non_indexed_col = 'updated3' WHERE id = 3; +SELECT pg_stat_force_next_flush(); + pg_stat_force_next_flush +-------------------------- + +(1 row) + +SELECT * FROM get_hot_count('hot_test'); + updates | hot +---------+----- + 3 | 3 +(1 row) + +-- Chain-of-1 on id=1 still has a predecessor line pointer. +WITH current_tuple AS (SELECT ctid FROM hot_test WHERE id = 1) +SELECT has_hot_chain('hot_test', current_tuple.ctid) AS has_chain, + chain_position, print_hot_chain.ctid, lp_flags, t_ctid +FROM current_tuple, LATERAL print_hot_chain('hot_test', current_tuple.ctid); + has_chain | chain_position | ctid | lp_flags | t_ctid +-----------+----------------+-------+------------+-------- + t | 0 | (0,1) | normal (1) | (0,4) + t | 1 | (0,4) | normal (1) | (0,4) +(2 rows) + +-- VACUUM collapses the chain. +VACUUM hot_test; +WITH current_tuple AS (SELECT ctid FROM hot_test WHERE id = 1) +SELECT has_hot_chain('hot_test', current_tuple.ctid) AS has_chain, + chain_position, print_hot_chain.ctid, lp_flags, t_ctid +FROM current_tuple, LATERAL print_hot_chain('hot_test', current_tuple.ctid); + has_chain | chain_position | ctid | lp_flags | t_ctid +-----------+----------------+-------+------------+-------- + f | 0 | (0,4) | normal (1) | (0,4) +(1 row) + +DROP TABLE hot_test; +-- --------------------------------------------------------------------------- +-- Summarizing indexes (BRIN) do not block HOT +-- --------------------------------------------------------------------------- +CREATE TABLE hot_test ( + id int PRIMARY KEY, + ts timestamp, + value int, + brin_col int +) WITH (fillfactor = 50); +CREATE INDEX hot_test_ts_brin ON hot_test USING brin(ts); +CREATE INDEX hot_test_brin_col_brin ON hot_test USING brin(brin_col); +INSERT INTO hot_test VALUES (1, '2024-01-01', 100, 1000); +-- BRIN columns are summarizing; updating them stays classic HOT even +-- though their values change. +UPDATE hot_test SET ts = '2024-01-02', brin_col = 2000 WHERE id = 1; +SELECT pg_stat_force_next_flush(); + pg_stat_force_next_flush +-------------------------- + +(1 row) + +SELECT * FROM get_hot_count('hot_test'); + updates | hot +---------+----- + 1 | 1 +(1 row) + +-- Non-indexed column: also HOT. +UPDATE hot_test SET value = 200 WHERE id = 1; +SELECT pg_stat_force_next_flush(); + pg_stat_force_next_flush +-------------------------- + +(1 row) + +SELECT * FROM get_hot_count('hot_test'); + updates | hot +---------+----- + 2 | 2 +(1 row) + +DROP TABLE hot_test; +-- --------------------------------------------------------------------------- +-- TOAST participates in HOT (non-indexed column paths only) +-- --------------------------------------------------------------------------- +CREATE TABLE hot_test ( + id int PRIMARY KEY, + indexed_col int, + large_text text, + small_text text +) WITH (fillfactor = 50); +CREATE INDEX hot_test_idx ON hot_test(indexed_col); +INSERT INTO hot_test VALUES (1, 100, repeat('x', 3000), 'small'); +-- Non-indexed, non-TOAST column: HOT. +UPDATE hot_test SET small_text = 'updated'; +SELECT pg_stat_force_next_flush(); + pg_stat_force_next_flush +-------------------------- + +(1 row) + +SELECT * FROM get_hot_count('hot_test'); + updates | hot +---------+----- + 1 | 1 +(1 row) + +-- TOAST column, indexed_col unchanged: HOT. +UPDATE hot_test SET large_text = repeat('y', 3000); +SELECT pg_stat_force_next_flush(); + pg_stat_force_next_flush +-------------------------- + +(1 row) + +SELECT * FROM get_hot_count('hot_test'); + updates | hot +---------+----- + 2 | 2 +(1 row) + +DROP TABLE hot_test; +-- --------------------------------------------------------------------------- +-- Partial index where update leaves indexed attrs unchanged +-- --------------------------------------------------------------------------- +CREATE TABLE hot_test ( + id int PRIMARY KEY, + status text, + data text +) WITH (fillfactor = 50); +CREATE INDEX hot_test_active_idx ON hot_test(status) WHERE status = 'active'; +INSERT INTO hot_test VALUES (1, 'active', 'data1'); +INSERT INTO hot_test VALUES (2, 'inactive', 'data2'); +INSERT INTO hot_test VALUES (3, 'deleted', 'data3'); +-- Update data on a row whose status matches the partial predicate: HOT. +UPDATE hot_test SET data = 'updated1' WHERE id = 1; +SELECT pg_stat_force_next_flush(); + pg_stat_force_next_flush +-------------------------- + +(1 row) + +SELECT * FROM get_hot_count('hot_test'); + updates | hot +---------+----- + 1 | 1 +(1 row) + +-- Update data on a row outside the predicate: HOT. +UPDATE hot_test SET data = 'updated2' WHERE id = 2; +SELECT pg_stat_force_next_flush(); + pg_stat_force_next_flush +-------------------------- + +(1 row) + +SELECT * FROM get_hot_count('hot_test'); + updates | hot +---------+----- + 2 | 2 +(1 row) + +SELECT id, status FROM hot_test WHERE status = 'active'; + id | status +----+-------- + 1 | active +(1 row) + +DROP TABLE hot_test; +-- --------------------------------------------------------------------------- +-- Multi-column btree: update of non-indexed column +-- --------------------------------------------------------------------------- +CREATE TABLE hot_test ( + id int PRIMARY KEY, + col_a int, + col_b int, + col_c int, + data text +) WITH (fillfactor = 50); +CREATE INDEX hot_test_ab_idx ON hot_test(col_a, col_b); +INSERT INTO hot_test VALUES (1, 10, 20, 30, 'data'); +-- col_c not in any index: HOT. +UPDATE hot_test SET col_c = 35; +-- data not in any index: HOT. +UPDATE hot_test SET data = 'updated'; +SELECT pg_stat_force_next_flush(); + pg_stat_force_next_flush +-------------------------- + +(1 row) + +SELECT * FROM get_hot_count('hot_test'); + updates | hot +---------+----- + 2 | 2 +(1 row) + +DROP TABLE hot_test; +-- --------------------------------------------------------------------------- +-- Unique index: update of non-indexed column + uniqueness enforcement +-- --------------------------------------------------------------------------- +CREATE TABLE hot_test ( + id int PRIMARY KEY, + unique_col int UNIQUE, + data text +) WITH (fillfactor = 50); +INSERT INTO hot_test VALUES (1, 100, 'data1'); +INSERT INTO hot_test VALUES (2, 200, 'data2'); +UPDATE hot_test SET data = 'updated'; +SELECT pg_stat_force_next_flush(); + pg_stat_force_next_flush +-------------------------- + +(1 row) + +SELECT * FROM get_hot_count('hot_test'); + updates | hot +---------+----- + 2 | 2 +(1 row) + +SELECT id, unique_col, data FROM hot_test ORDER BY id; + id | unique_col | data +----+------------+--------- + 1 | 100 | updated + 2 | 200 | updated +(2 rows) + +-- Unique constraint still enforced on any path. +UPDATE hot_test SET unique_col = 100 WHERE id = 2; +ERROR: duplicate key value violates unique constraint "hot_test_unique_col_key" +DETAIL: Key (unique_col)=(100) already exists. +DROP TABLE hot_test; +-- --------------------------------------------------------------------------- +-- Partitioned tables: HOT within a partition +-- --------------------------------------------------------------------------- +CREATE TABLE hot_test_partitioned ( + id int, + partition_key int, + indexed_col int, + data text, + PRIMARY KEY (id, partition_key) +) PARTITION BY RANGE (partition_key); +CREATE TABLE hot_test_part1 PARTITION OF hot_test_partitioned + FOR VALUES FROM (1) TO (100) WITH (fillfactor = 50); +CREATE TABLE hot_test_part2 PARTITION OF hot_test_partitioned + FOR VALUES FROM (100) TO (200) WITH (fillfactor = 50); +CREATE INDEX hot_test_part_idx ON hot_test_partitioned(indexed_col); +INSERT INTO hot_test_partitioned VALUES (1, 50, 100, 'initial1'); +INSERT INTO hot_test_partitioned VALUES (2, 150, 200, 'initial2'); +UPDATE hot_test_partitioned SET data = 'updated1' WHERE id = 1; +UPDATE hot_test_partitioned SET data = 'updated2' WHERE id = 2; +SELECT pg_stat_force_next_flush(); + pg_stat_force_next_flush +-------------------------- + +(1 row) + +SELECT * FROM get_hot_count('hot_test_part1'); + updates | hot +---------+----- + 1 | 1 +(1 row) + +SELECT pg_stat_force_next_flush(); + pg_stat_force_next_flush +-------------------------- + +(1 row) + +SELECT * FROM get_hot_count('hot_test_part2'); + updates | hot +---------+----- + 1 | 1 +(1 row) + +SELECT id FROM hot_test_partitioned WHERE indexed_col = 100; + id +---- + 1 +(1 row) + +SELECT id FROM hot_test_partitioned WHERE indexed_col = 200; + id +---- + 2 +(1 row) + +DROP TABLE hot_test_partitioned CASCADE; +-- --------------------------------------------------------------------------- +-- Expression index on a JSONB field +-- --------------------------------------------------------------------------- +CREATE TABLE hot_jsonb_test ( + id int PRIMARY KEY, + data jsonb +) WITH (fillfactor = 50); +CREATE INDEX hot_jsonb_name_idx ON hot_jsonb_test ((data->>'name')); +INSERT INTO hot_jsonb_test VALUES + (1, '{"name":"Alice","age":30,"city":"NYC"}'), + (2, '{"name":"Bob","age":25,"city":"LA"}'); +-- Each UPDATE changes the indexed column but not the indexed expression. +UPDATE hot_jsonb_test SET data = jsonb_set(data, '{age}', '31') WHERE id = 1; +SELECT pg_stat_force_next_flush(); + pg_stat_force_next_flush +-------------------------- + +(1 row) + +SELECT * FROM get_hot_count('hot_jsonb_test'); + updates | hot +---------+----- + 1 | 0 +(1 row) + +UPDATE hot_jsonb_test SET data = data - 'city' WHERE id = 2; +SELECT pg_stat_force_next_flush(); + pg_stat_force_next_flush +-------------------------- + +(1 row) + +SELECT * FROM get_hot_count('hot_jsonb_test'); + updates | hot +---------+----- + 2 | 0 +(1 row) + +UPDATE hot_jsonb_test SET data = jsonb_insert(data, '{country}', '"USA"') WHERE id = 2; +SELECT pg_stat_force_next_flush(); + pg_stat_force_next_flush +-------------------------- + +(1 row) + +SELECT * FROM get_hot_count('hot_jsonb_test'); + updates | hot +---------+----- + 3 | 0 +(1 row) + +DROP TABLE hot_jsonb_test; +-- --------------------------------------------------------------------------- +-- Expression and partial indexes +-- +-- update_case() runs one UPDATE on a fresh table and reports how many rows +-- it updated, and how many of those with HOT. +-- --------------------------------------------------------------------------- +CREATE FUNCTION update_case(setup text, upd text, + OUT updated bigint, OUT hot bigint) +LANGUAGE plpgsql AS $$ +BEGIN + EXECUTE 'DROP TABLE IF EXISTS hot_expr_test'; + EXECUTE setup; + updated := pg_stat_get_xact_tuples_updated('hot_expr_test'::regclass); + hot := pg_stat_get_xact_tuples_hot_updated('hot_expr_test'::regclass); + EXECUTE upd; + updated := pg_stat_get_xact_tuples_updated('hot_expr_test'::regclass) - updated; + hot := pg_stat_get_xact_tuples_hot_updated('hot_expr_test'::regclass) - hot; +END; +$$; +-- Expression value unchanged. +SELECT * FROM update_case( + $$CREATE TABLE hot_expr_test (id int PRIMARY KEY, s text) WITH (fillfactor = 50); + CREATE INDEX ON hot_expr_test (lower(s)); + INSERT INTO hot_expr_test SELECT g, 'abc' || g FROM generate_series(1, 10) g$$, + $$UPDATE hot_expr_test SET s = upper(s)$$); +NOTICE: table "hot_expr_test" does not exist, skipping + updated | hot +---------+----- + 10 | 0 +(1 row) + +SELECT * FROM update_case( + $$CREATE TABLE hot_expr_test (id int PRIMARY KEY, a int, b int) WITH (fillfactor = 50); + CREATE INDEX ON hot_expr_test ((a + b)); + INSERT INTO hot_expr_test SELECT g, g, 0 FROM generate_series(1, 10) g$$, + $$UPDATE hot_expr_test SET a = a - 1, b = b + 1$$); + updated | hot +---------+----- + 10 | 0 +(1 row) + +-- Expression value changed. +SELECT * FROM update_case( + $$CREATE TABLE hot_expr_test (id int PRIMARY KEY, d jsonb) WITH (fillfactor = 50); + CREATE INDEX ON hot_expr_test ((d->>'name')); + INSERT INTO hot_expr_test SELECT g, jsonb_build_object('name', 'n' || g) + FROM generate_series(1, 10) g$$, + $$UPDATE hot_expr_test SET d = jsonb_set(d, '{name}', '"x"')$$); + updated | hot +---------+----- + 10 | 0 +(1 row) + +-- Equal values that are stored differently. +SELECT * FROM update_case( + $$CREATE TABLE hot_expr_test (id int PRIMARY KEY, n numeric) WITH (fillfactor = 50); + CREATE INDEX ON hot_expr_test ((n + 0)); + INSERT INTO hot_expr_test SELECT g, 3 FROM generate_series(1, 10) g$$, + $$UPDATE hot_expr_test SET n = 3.0$$); + updated | hot +---------+----- + 10 | 0 +(1 row) + +-- The index stores a type other than the expression's: name_ops stores +-- cstring, and gist point_ops stores box. +SELECT * FROM update_case( + $$CREATE TABLE hot_expr_test (id int PRIMARY KEY, s text) WITH (fillfactor = 50); + CREATE INDEX ON hot_expr_test ((lower(s)::name)); + INSERT INTO hot_expr_test SELECT g, 'abc' || g FROM generate_series(1, 10) g$$, + $$UPDATE hot_expr_test SET s = upper(s)$$); + updated | hot +---------+----- + 10 | 0 +(1 row) + +SELECT * FROM update_case( + $$CREATE TABLE hot_expr_test (id int PRIMARY KEY, p point) WITH (fillfactor = 50); + CREATE INDEX ON hot_expr_test USING gist ((point(p[0], 0))); + INSERT INTO hot_expr_test SELECT g, point(g, g) FROM generate_series(1, 10) g$$, + $$UPDATE hot_expr_test SET p = point(p[0], p[1] + 1)$$); + updated | hot +---------+----- + 10 | 0 +(1 row) + +-- An expression that is a point, stored by the gist index as a box. This +-- update crashed the server in PostgreSQL 11.0. +SELECT * FROM update_case( + $$CREATE TABLE hot_expr_test (id int PRIMARY KEY, d smallint) WITH (fillfactor = 50); + CREATE INDEX ON hot_expr_test USING gist ((point(d::integer % 7, 0))); + INSERT INTO hot_expr_test SELECT g, 1 FROM generate_series(1, 10) g$$, + $$UPDATE hot_expr_test SET d = d + 7$$); + updated | hot +---------+----- + 10 | 0 +(1 row) + +-- A plain index on the same column as an expression index. +SELECT * FROM update_case( + $$CREATE TABLE hot_expr_test (id int PRIMARY KEY, s text) WITH (fillfactor = 50); + CREATE INDEX ON hot_expr_test (lower(s)); + CREATE INDEX ON hot_expr_test (s); + INSERT INTO hot_expr_test SELECT g, 'abc' || g FROM generate_series(1, 10) g$$, + $$UPDATE hot_expr_test SET s = upper(s)$$); + updated | hot +---------+----- + 10 | 0 +(1 row) + +-- A BRIN index whose column changed, and an expression index whose value +-- did not. +SELECT * FROM update_case( + $$CREATE TABLE hot_expr_test (id int PRIMARY KEY, s text, b int) WITH (fillfactor = 50); + CREATE INDEX ON hot_expr_test (lower(s)); + CREATE INDEX ON hot_expr_test USING brin (b); + INSERT INTO hot_expr_test SELECT g, 'abc' || g, g FROM generate_series(1, 10) g$$, + $$UPDATE hot_expr_test SET s = upper(s), b = b + 1$$); + updated | hot +---------+----- + 10 | 0 +(1 row) + +-- A BEFORE ROW trigger that changes the expression's value. +CREATE FUNCTION hot_expr_trig() RETURNS trigger LANGUAGE plpgsql AS + $$BEGIN NEW.s := NEW.s || 'z'; RETURN NEW; END$$; +SELECT * FROM update_case( + $$CREATE TABLE hot_expr_test (id int PRIMARY KEY, s text, x int) WITH (fillfactor = 50); + CREATE INDEX ON hot_expr_test (lower(s)); + CREATE TRIGGER t BEFORE UPDATE ON hot_expr_test + FOR EACH ROW EXECUTE FUNCTION hot_expr_trig(); + INSERT INTO hot_expr_test SELECT g, 'abc' || g, 0 FROM generate_series(1, 10) g$$, + $$UPDATE hot_expr_test SET x = 1$$); + updated | hot +---------+----- + 10 | 0 +(1 row) + +DROP FUNCTION hot_expr_trig() CASCADE; +NOTICE: drop cascades to trigger t on table hot_expr_test +-- Partial index, both versions outside the predicate, changing the key or +-- the predicate's column. +SELECT * FROM update_case( + $$CREATE TABLE hot_expr_test (id int PRIMARY KEY, status text, v int) WITH (fillfactor = 50); + CREATE INDEX ON hot_expr_test (v) WHERE status = 'active'; + INSERT INTO hot_expr_test SELECT g, 'done', g FROM generate_series(1, 10) g$$, + $$UPDATE hot_expr_test SET v = v + 1$$); + updated | hot +---------+----- + 10 | 0 +(1 row) + +SELECT * FROM update_case( + $$CREATE TABLE hot_expr_test (id int PRIMARY KEY, status text, v int) WITH (fillfactor = 50); + CREATE INDEX ON hot_expr_test (v) WHERE status = 'active'; + INSERT INTO hot_expr_test SELECT g, 'done', g FROM generate_series(1, 10) g$$, + $$UPDATE hot_expr_test SET status = 'archived'$$); + updated | hot +---------+----- + 10 | 0 +(1 row) + +-- Both versions inside the predicate with the key unchanged. +SELECT * FROM update_case( + $$CREATE TABLE hot_expr_test (id int PRIMARY KEY, status text, v int) WITH (fillfactor = 50); + CREATE INDEX ON hot_expr_test (v) WHERE status <> 'x'; + INSERT INTO hot_expr_test SELECT g, 'a', g FROM generate_series(1, 10) g$$, + $$UPDATE hot_expr_test SET status = 'b'$$); + updated | hot +---------+----- + 10 | 0 +(1 row) + +-- Entering or leaving the predicate, or changing the key inside it. +SELECT * FROM update_case( + $$CREATE TABLE hot_expr_test (id int PRIMARY KEY, status text, v int) WITH (fillfactor = 50); + CREATE INDEX ON hot_expr_test (v) WHERE status = 'active'; + INSERT INTO hot_expr_test SELECT g, 'active', g FROM generate_series(1, 10) g$$, + $$UPDATE hot_expr_test SET status = 'done'$$); + updated | hot +---------+----- + 10 | 0 +(1 row) + +SELECT * FROM update_case( + $$CREATE TABLE hot_expr_test (id int PRIMARY KEY, status text, v int) WITH (fillfactor = 50); + CREATE INDEX ON hot_expr_test (v) WHERE status = 'active'; + INSERT INTO hot_expr_test SELECT g, 'done', g FROM generate_series(1, 10) g$$, + $$UPDATE hot_expr_test SET status = 'active'$$); + updated | hot +---------+----- + 10 | 0 +(1 row) + +SELECT * FROM update_case( + $$CREATE TABLE hot_expr_test (id int PRIMARY KEY, status text, v int) WITH (fillfactor = 50); + CREATE INDEX ON hot_expr_test (v) WHERE status = 'active'; + INSERT INTO hot_expr_test SELECT g, 'active', g FROM generate_series(1, 10) g$$, + $$UPDATE hot_expr_test SET v = v + 1000$$); + updated | hot +---------+----- + 10 | 0 +(1 row) + +-- Rows updated outside a partial index must not be found through it. +SELECT * FROM update_case( + $$CREATE TABLE hot_expr_test (id int PRIMARY KEY, status text, v int) WITH (fillfactor = 50); + CREATE INDEX hot_expr_test_v ON hot_expr_test (v) WHERE status = 'active'; + INSERT INTO hot_expr_test SELECT g, CASE WHEN g <= 5 THEN 'active' ELSE 'done' END, g + FROM generate_series(1, 10) g$$, + $$UPDATE hot_expr_test SET v = v + 100 WHERE status = 'done'$$); + updated | hot +---------+----- + 5 | 0 +(1 row) + +SET enable_seqscan = off; +SET enable_bitmapscan = off; +EXPLAIN (COSTS OFF) SELECT id FROM hot_expr_test WHERE status = 'active' AND v > 0; + QUERY PLAN +--------------------------------------------------- + Index Scan using hot_expr_test_v on hot_expr_test + Index Cond: (v > 0) +(2 rows) + +SELECT id FROM hot_expr_test WHERE status = 'active' AND v > 0 ORDER BY v; + id +---- + 1 + 2 + 3 + 4 + 5 +(5 rows) + +RESET enable_seqscan; +RESET enable_bitmapscan; +SELECT id FROM hot_expr_test WHERE status = 'active' AND v > 0 ORDER BY v; + id +---- + 1 + 2 + 3 + 4 + 5 +(5 rows) + +DROP TABLE hot_expr_test; +-- One UPDATE whose rows each need a different index to decide: some change +-- the expression index, some only enter or leave the partial index created +-- after it, some change neither. The partial index must hold exactly the +-- rows that satisfy its predicate. +CREATE TABLE hot_expr_multi (id int PRIMARY KEY, doc jsonb) WITH (fillfactor = 30); +CREATE INDEX hot_expr_multi_status ON hot_expr_multi ((doc->>'status')); +CREATE INDEX hot_expr_multi_vip ON hot_expr_multi ((doc->>'email')) + WHERE (doc->>'vip')::bool; +INSERT INTO hot_expr_multi + SELECT g, jsonb_build_object('status', 'a', 'email', 'e' || g, + 'vip', g % 2 = 0, 'n', 0) + FROM generate_series(1, 300) g; +UPDATE hot_expr_multi SET doc = CASE + WHEN id % 4 = 0 THEN jsonb_set(doc, '{status}', '"b"') + WHEN id % 4 = 1 THEN jsonb_set(doc, '{vip}', to_jsonb(NOT (doc->>'vip')::bool)) + WHEN id % 4 = 2 THEN jsonb_set(jsonb_set(doc, '{status}', '"c"'), + '{vip}', to_jsonb(NOT (doc->>'vip')::bool)) + ELSE jsonb_set(doc, '{n}', '1') END; +SET enable_seqscan = off; +SET enable_bitmapscan = off; +SELECT count(*) FROM hot_expr_multi + WHERE (doc->>'vip')::bool AND doc->>'email' >= 'e'; + count +------- + 150 +(1 row) + +RESET enable_seqscan; +RESET enable_bitmapscan; +SELECT count(*) FROM hot_expr_multi WHERE (doc->>'vip')::bool; + count +------- + 150 +(1 row) + +DROP TABLE hot_expr_multi; +-- An index whose concurrent build failed is not valid, and its expression +-- need not be computable for every row. An UPDATE must not evaluate it. +CREATE TABLE hot_expr_cic (id int PRIMARY KEY, x int) WITH (fillfactor = 50); +INSERT INTO hot_expr_cic VALUES (1, 1), (2, 0); +CREATE INDEX CONCURRENTLY hot_expr_cic_idx ON hot_expr_cic ((100 / x)); +ERROR: division by zero +SELECT indisready, indisvalid FROM pg_index + WHERE indexrelid = 'hot_expr_cic_idx'::regclass; + indisready | indisvalid +------------+------------ + f | f +(1 row) + +UPDATE hot_expr_cic SET x = 0 WHERE id = 1; +UPDATE hot_expr_cic SET x = 1 WHERE id = 2; +SELECT * FROM hot_expr_cic ORDER BY id; + id | x +----+--- + 1 | 0 + 2 | 1 +(2 rows) + +DROP TABLE hot_expr_cic; +DROP FUNCTION update_case(text, text); +-- --------------------------------------------------------------------------- +-- Cleanup +-- --------------------------------------------------------------------------- +DROP FUNCTION has_hot_chain(text, tid); +DROP FUNCTION print_hot_chain(text, tid); +DROP FUNCTION get_hot_count(text); +-- Leave pageinspect installed: this runs inside the pageinspect regress suite +-- (page.sql created it), and other tests in the schedule still need it. diff --git a/contrib/pageinspect/meson.build b/contrib/pageinspect/meson.build index c43ea400a4d..105a5b7d68e 100644 --- a/contrib/pageinspect/meson.build +++ b/contrib/pageinspect/meson.build @@ -56,6 +56,7 @@ tests += { 'hash', 'checksum', 'oldextversions', + 'hot_updates', ], }, } diff --git a/contrib/pageinspect/sql/hot_updates.sql b/contrib/pageinspect/sql/hot_updates.sql new file mode 100644 index 00000000000..558d8d742e9 --- /dev/null +++ b/contrib/pageinspect/sql/hot_updates.sql @@ -0,0 +1,562 @@ +-- +-- HOT_UPDATES +-- Test Heap-Only Tuple (HOT) update decisions +-- +-- Expected output records which UPDATEs are HOT, so that a change to the +-- rules shows up as a change to this file. +-- +-- Validation methods: +-- 1. Statistics (pg_stat_get_tuples_hot_updated) +-- 2. pageinspect for HOT chain structure +-- 3. EXPLAIN to confirm the planner still picks the index +-- + +-- Load required extensions +CREATE EXTENSION IF NOT EXISTS pageinspect; + +-- Sum of committed and in-progress (non-HOT, HOT) update counters. +CREATE OR REPLACE FUNCTION get_hot_count(rel_name text) +RETURNS TABLE ( + updates BIGINT, + hot BIGINT +) AS $$ +DECLARE + rel_oid oid; +BEGIN + rel_oid := rel_name::regclass::oid; + updates := COALESCE(pg_stat_get_tuples_updated(rel_oid), 0) + + COALESCE(pg_stat_get_xact_tuples_updated(rel_oid), 0); + hot := COALESCE(pg_stat_get_tuples_hot_updated(rel_oid), 0) + + COALESCE(pg_stat_get_xact_tuples_hot_updated(rel_oid), 0); + RETURN NEXT; +END; +$$ LANGUAGE plpgsql; + +-- True iff target_ctid is the TAIL of a HOT chain on the same page. +CREATE OR REPLACE FUNCTION has_hot_chain(rel_name text, target_ctid tid) +RETURNS boolean AS $$ +DECLARE + block_num int; + page_item record; +BEGIN + block_num := (target_ctid::text::point)[0]::int; + FOR page_item IN + SELECT lp, lp_flags, t_ctid + FROM heap_page_items(get_raw_page(rel_name, block_num)) + WHERE lp_flags = 1 + AND t_ctid IS NOT NULL + AND t_ctid = target_ctid + AND ('(' || block_num::text || ',' || lp::text || ')')::tid != target_ctid + LOOP + RETURN true; + END LOOP; + RETURN false; +END; +$$ LANGUAGE plpgsql; + +-- Emit the HOT chain rooted at start_ctid. +CREATE OR REPLACE FUNCTION print_hot_chain(rel_name text, start_ctid tid) +RETURNS TABLE(chain_position int, ctid tid, lp_flags text, t_ctid tid, chain_end boolean) AS +$$ +#variable_conflict use_column +DECLARE + block_num int; + line_ptr int; + current_ctid tid := start_ctid; + next_ctid tid; + position int := 0; + max_iterations int := 100; + page_item record; + found_predecessor boolean := false; + flags_name text; +BEGIN + block_num := (start_ctid::text::point)[0]::int; + + FOR page_item IN + SELECT lp, lp_flags, t_ctid + FROM heap_page_items(get_raw_page(rel_name, block_num)) + WHERE lp_flags = 1 + AND t_ctid = start_ctid + LOOP + current_ctid := ('(' || block_num::text || ',' || page_item.lp::text || ')')::tid; + found_predecessor := true; + EXIT; + END LOOP; + IF NOT found_predecessor THEN + current_ctid := start_ctid; + END IF; + + WHILE position < max_iterations LOOP + line_ptr := (current_ctid::text::point)[1]::int; + FOR page_item IN + SELECT lp, lp_flags, t_ctid + FROM heap_page_items(get_raw_page(rel_name, block_num)) + WHERE lp = line_ptr + LOOP + flags_name := CASE page_item.lp_flags + WHEN 0 THEN 'unused (0)' + WHEN 1 THEN 'normal (1)' + WHEN 2 THEN 'redirect (2)' + WHEN 3 THEN 'dead (3)' + ELSE 'unknown (' || page_item.lp_flags::text || ')' + END; + RETURN QUERY SELECT + position, + current_ctid, + flags_name, + page_item.t_ctid, + (page_item.t_ctid IS NULL OR page_item.t_ctid = current_ctid)::boolean; + + IF page_item.t_ctid IS NULL OR page_item.t_ctid = current_ctid THEN + RETURN; + END IF; + next_ctid := page_item.t_ctid; + IF (next_ctid::text::point)[0]::int != block_num THEN + RETURN; + END IF; + current_ctid := next_ctid; + position := position + 1; + END LOOP; + IF position = 0 THEN + RETURN; + END IF; + END LOOP; +END; +$$ LANGUAGE plpgsql; + + +-- --------------------------------------------------------------------------- +-- Basic HOT: update of a non-indexed column +-- --------------------------------------------------------------------------- +CREATE TABLE hot_test ( + id int PRIMARY KEY, + indexed_col int, + non_indexed_col text +) WITH (fillfactor = 50); +CREATE INDEX hot_test_indexed_idx ON hot_test(indexed_col); + +INSERT INTO hot_test VALUES (1, 100, 'initial'); +INSERT INTO hot_test VALUES (2, 200, 'initial'); +INSERT INTO hot_test VALUES (3, 300, 'initial'); + +SELECT pg_stat_force_next_flush(); +SELECT * FROM get_hot_count('hot_test'); + +-- Three classic HOT updates (non-indexed col). +UPDATE hot_test SET non_indexed_col = 'updated1' WHERE id = 1; +UPDATE hot_test SET non_indexed_col = 'updated2' WHERE id = 2; +UPDATE hot_test SET non_indexed_col = 'updated3' WHERE id = 3; +SELECT pg_stat_force_next_flush(); +SELECT * FROM get_hot_count('hot_test'); + +-- Chain-of-1 on id=1 still has a predecessor line pointer. +WITH current_tuple AS (SELECT ctid FROM hot_test WHERE id = 1) +SELECT has_hot_chain('hot_test', current_tuple.ctid) AS has_chain, + chain_position, print_hot_chain.ctid, lp_flags, t_ctid +FROM current_tuple, LATERAL print_hot_chain('hot_test', current_tuple.ctid); + +-- VACUUM collapses the chain. +VACUUM hot_test; + +WITH current_tuple AS (SELECT ctid FROM hot_test WHERE id = 1) +SELECT has_hot_chain('hot_test', current_tuple.ctid) AS has_chain, + chain_position, print_hot_chain.ctid, lp_flags, t_ctid +FROM current_tuple, LATERAL print_hot_chain('hot_test', current_tuple.ctid); + +DROP TABLE hot_test; + +-- --------------------------------------------------------------------------- +-- Summarizing indexes (BRIN) do not block HOT +-- --------------------------------------------------------------------------- +CREATE TABLE hot_test ( + id int PRIMARY KEY, + ts timestamp, + value int, + brin_col int +) WITH (fillfactor = 50); +CREATE INDEX hot_test_ts_brin ON hot_test USING brin(ts); +CREATE INDEX hot_test_brin_col_brin ON hot_test USING brin(brin_col); + +INSERT INTO hot_test VALUES (1, '2024-01-01', 100, 1000); + +-- BRIN columns are summarizing; updating them stays classic HOT even +-- though their values change. +UPDATE hot_test SET ts = '2024-01-02', brin_col = 2000 WHERE id = 1; +SELECT pg_stat_force_next_flush(); +SELECT * FROM get_hot_count('hot_test'); + +-- Non-indexed column: also HOT. +UPDATE hot_test SET value = 200 WHERE id = 1; +SELECT pg_stat_force_next_flush(); +SELECT * FROM get_hot_count('hot_test'); + +DROP TABLE hot_test; + +-- --------------------------------------------------------------------------- +-- TOAST participates in HOT (non-indexed column paths only) +-- --------------------------------------------------------------------------- +CREATE TABLE hot_test ( + id int PRIMARY KEY, + indexed_col int, + large_text text, + small_text text +) WITH (fillfactor = 50); +CREATE INDEX hot_test_idx ON hot_test(indexed_col); + +INSERT INTO hot_test VALUES (1, 100, repeat('x', 3000), 'small'); + +-- Non-indexed, non-TOAST column: HOT. +UPDATE hot_test SET small_text = 'updated'; +SELECT pg_stat_force_next_flush(); +SELECT * FROM get_hot_count('hot_test'); + +-- TOAST column, indexed_col unchanged: HOT. +UPDATE hot_test SET large_text = repeat('y', 3000); +SELECT pg_stat_force_next_flush(); +SELECT * FROM get_hot_count('hot_test'); + +DROP TABLE hot_test; + +-- --------------------------------------------------------------------------- +-- Partial index where update leaves indexed attrs unchanged +-- --------------------------------------------------------------------------- +CREATE TABLE hot_test ( + id int PRIMARY KEY, + status text, + data text +) WITH (fillfactor = 50); +CREATE INDEX hot_test_active_idx ON hot_test(status) WHERE status = 'active'; + +INSERT INTO hot_test VALUES (1, 'active', 'data1'); +INSERT INTO hot_test VALUES (2, 'inactive', 'data2'); +INSERT INTO hot_test VALUES (3, 'deleted', 'data3'); + +-- Update data on a row whose status matches the partial predicate: HOT. +UPDATE hot_test SET data = 'updated1' WHERE id = 1; +SELECT pg_stat_force_next_flush(); +SELECT * FROM get_hot_count('hot_test'); + +-- Update data on a row outside the predicate: HOT. +UPDATE hot_test SET data = 'updated2' WHERE id = 2; +SELECT pg_stat_force_next_flush(); +SELECT * FROM get_hot_count('hot_test'); + +SELECT id, status FROM hot_test WHERE status = 'active'; + +DROP TABLE hot_test; + +-- --------------------------------------------------------------------------- +-- Multi-column btree: update of non-indexed column +-- --------------------------------------------------------------------------- +CREATE TABLE hot_test ( + id int PRIMARY KEY, + col_a int, + col_b int, + col_c int, + data text +) WITH (fillfactor = 50); +CREATE INDEX hot_test_ab_idx ON hot_test(col_a, col_b); + +INSERT INTO hot_test VALUES (1, 10, 20, 30, 'data'); + +-- col_c not in any index: HOT. +UPDATE hot_test SET col_c = 35; +-- data not in any index: HOT. +UPDATE hot_test SET data = 'updated'; +SELECT pg_stat_force_next_flush(); +SELECT * FROM get_hot_count('hot_test'); + +DROP TABLE hot_test; + +-- --------------------------------------------------------------------------- +-- Unique index: update of non-indexed column + uniqueness enforcement +-- --------------------------------------------------------------------------- +CREATE TABLE hot_test ( + id int PRIMARY KEY, + unique_col int UNIQUE, + data text +) WITH (fillfactor = 50); + +INSERT INTO hot_test VALUES (1, 100, 'data1'); +INSERT INTO hot_test VALUES (2, 200, 'data2'); + +UPDATE hot_test SET data = 'updated'; +SELECT pg_stat_force_next_flush(); +SELECT * FROM get_hot_count('hot_test'); + +SELECT id, unique_col, data FROM hot_test ORDER BY id; + +-- Unique constraint still enforced on any path. +UPDATE hot_test SET unique_col = 100 WHERE id = 2; + +DROP TABLE hot_test; + +-- --------------------------------------------------------------------------- +-- Partitioned tables: HOT within a partition +-- --------------------------------------------------------------------------- +CREATE TABLE hot_test_partitioned ( + id int, + partition_key int, + indexed_col int, + data text, + PRIMARY KEY (id, partition_key) +) PARTITION BY RANGE (partition_key); + +CREATE TABLE hot_test_part1 PARTITION OF hot_test_partitioned + FOR VALUES FROM (1) TO (100) WITH (fillfactor = 50); +CREATE TABLE hot_test_part2 PARTITION OF hot_test_partitioned + FOR VALUES FROM (100) TO (200) WITH (fillfactor = 50); + +CREATE INDEX hot_test_part_idx ON hot_test_partitioned(indexed_col); + +INSERT INTO hot_test_partitioned VALUES (1, 50, 100, 'initial1'); +INSERT INTO hot_test_partitioned VALUES (2, 150, 200, 'initial2'); + +UPDATE hot_test_partitioned SET data = 'updated1' WHERE id = 1; +UPDATE hot_test_partitioned SET data = 'updated2' WHERE id = 2; + +SELECT pg_stat_force_next_flush(); +SELECT * FROM get_hot_count('hot_test_part1'); +SELECT pg_stat_force_next_flush(); +SELECT * FROM get_hot_count('hot_test_part2'); + +SELECT id FROM hot_test_partitioned WHERE indexed_col = 100; +SELECT id FROM hot_test_partitioned WHERE indexed_col = 200; + +DROP TABLE hot_test_partitioned CASCADE; + +-- --------------------------------------------------------------------------- +-- Expression index on a JSONB field +-- --------------------------------------------------------------------------- +CREATE TABLE hot_jsonb_test ( + id int PRIMARY KEY, + data jsonb +) WITH (fillfactor = 50); +CREATE INDEX hot_jsonb_name_idx ON hot_jsonb_test ((data->>'name')); + +INSERT INTO hot_jsonb_test VALUES + (1, '{"name":"Alice","age":30,"city":"NYC"}'), + (2, '{"name":"Bob","age":25,"city":"LA"}'); + +-- Each UPDATE changes the indexed column but not the indexed expression. +UPDATE hot_jsonb_test SET data = jsonb_set(data, '{age}', '31') WHERE id = 1; +SELECT pg_stat_force_next_flush(); +SELECT * FROM get_hot_count('hot_jsonb_test'); + +UPDATE hot_jsonb_test SET data = data - 'city' WHERE id = 2; +SELECT pg_stat_force_next_flush(); +SELECT * FROM get_hot_count('hot_jsonb_test'); + +UPDATE hot_jsonb_test SET data = jsonb_insert(data, '{country}', '"USA"') WHERE id = 2; +SELECT pg_stat_force_next_flush(); +SELECT * FROM get_hot_count('hot_jsonb_test'); + +DROP TABLE hot_jsonb_test; + + +-- --------------------------------------------------------------------------- +-- Expression and partial indexes +-- +-- update_case() runs one UPDATE on a fresh table and reports how many rows +-- it updated, and how many of those with HOT. +-- --------------------------------------------------------------------------- +CREATE FUNCTION update_case(setup text, upd text, + OUT updated bigint, OUT hot bigint) +LANGUAGE plpgsql AS $$ +BEGIN + EXECUTE 'DROP TABLE IF EXISTS hot_expr_test'; + EXECUTE setup; + updated := pg_stat_get_xact_tuples_updated('hot_expr_test'::regclass); + hot := pg_stat_get_xact_tuples_hot_updated('hot_expr_test'::regclass); + EXECUTE upd; + updated := pg_stat_get_xact_tuples_updated('hot_expr_test'::regclass) - updated; + hot := pg_stat_get_xact_tuples_hot_updated('hot_expr_test'::regclass) - hot; +END; +$$; + +-- Expression value unchanged. +SELECT * FROM update_case( + $$CREATE TABLE hot_expr_test (id int PRIMARY KEY, s text) WITH (fillfactor = 50); + CREATE INDEX ON hot_expr_test (lower(s)); + INSERT INTO hot_expr_test SELECT g, 'abc' || g FROM generate_series(1, 10) g$$, + $$UPDATE hot_expr_test SET s = upper(s)$$); + +SELECT * FROM update_case( + $$CREATE TABLE hot_expr_test (id int PRIMARY KEY, a int, b int) WITH (fillfactor = 50); + CREATE INDEX ON hot_expr_test ((a + b)); + INSERT INTO hot_expr_test SELECT g, g, 0 FROM generate_series(1, 10) g$$, + $$UPDATE hot_expr_test SET a = a - 1, b = b + 1$$); + +-- Expression value changed. +SELECT * FROM update_case( + $$CREATE TABLE hot_expr_test (id int PRIMARY KEY, d jsonb) WITH (fillfactor = 50); + CREATE INDEX ON hot_expr_test ((d->>'name')); + INSERT INTO hot_expr_test SELECT g, jsonb_build_object('name', 'n' || g) + FROM generate_series(1, 10) g$$, + $$UPDATE hot_expr_test SET d = jsonb_set(d, '{name}', '"x"')$$); + +-- Equal values that are stored differently. +SELECT * FROM update_case( + $$CREATE TABLE hot_expr_test (id int PRIMARY KEY, n numeric) WITH (fillfactor = 50); + CREATE INDEX ON hot_expr_test ((n + 0)); + INSERT INTO hot_expr_test SELECT g, 3 FROM generate_series(1, 10) g$$, + $$UPDATE hot_expr_test SET n = 3.0$$); + +-- The index stores a type other than the expression's: name_ops stores +-- cstring, and gist point_ops stores box. +SELECT * FROM update_case( + $$CREATE TABLE hot_expr_test (id int PRIMARY KEY, s text) WITH (fillfactor = 50); + CREATE INDEX ON hot_expr_test ((lower(s)::name)); + INSERT INTO hot_expr_test SELECT g, 'abc' || g FROM generate_series(1, 10) g$$, + $$UPDATE hot_expr_test SET s = upper(s)$$); + +SELECT * FROM update_case( + $$CREATE TABLE hot_expr_test (id int PRIMARY KEY, p point) WITH (fillfactor = 50); + CREATE INDEX ON hot_expr_test USING gist ((point(p[0], 0))); + INSERT INTO hot_expr_test SELECT g, point(g, g) FROM generate_series(1, 10) g$$, + $$UPDATE hot_expr_test SET p = point(p[0], p[1] + 1)$$); + +-- An expression that is a point, stored by the gist index as a box. This +-- update crashed the server in PostgreSQL 11.0. +SELECT * FROM update_case( + $$CREATE TABLE hot_expr_test (id int PRIMARY KEY, d smallint) WITH (fillfactor = 50); + CREATE INDEX ON hot_expr_test USING gist ((point(d::integer % 7, 0))); + INSERT INTO hot_expr_test SELECT g, 1 FROM generate_series(1, 10) g$$, + $$UPDATE hot_expr_test SET d = d + 7$$); + +-- A plain index on the same column as an expression index. +SELECT * FROM update_case( + $$CREATE TABLE hot_expr_test (id int PRIMARY KEY, s text) WITH (fillfactor = 50); + CREATE INDEX ON hot_expr_test (lower(s)); + CREATE INDEX ON hot_expr_test (s); + INSERT INTO hot_expr_test SELECT g, 'abc' || g FROM generate_series(1, 10) g$$, + $$UPDATE hot_expr_test SET s = upper(s)$$); + +-- A BRIN index whose column changed, and an expression index whose value +-- did not. +SELECT * FROM update_case( + $$CREATE TABLE hot_expr_test (id int PRIMARY KEY, s text, b int) WITH (fillfactor = 50); + CREATE INDEX ON hot_expr_test (lower(s)); + CREATE INDEX ON hot_expr_test USING brin (b); + INSERT INTO hot_expr_test SELECT g, 'abc' || g, g FROM generate_series(1, 10) g$$, + $$UPDATE hot_expr_test SET s = upper(s), b = b + 1$$); + +-- A BEFORE ROW trigger that changes the expression's value. +CREATE FUNCTION hot_expr_trig() RETURNS trigger LANGUAGE plpgsql AS + $$BEGIN NEW.s := NEW.s || 'z'; RETURN NEW; END$$; +SELECT * FROM update_case( + $$CREATE TABLE hot_expr_test (id int PRIMARY KEY, s text, x int) WITH (fillfactor = 50); + CREATE INDEX ON hot_expr_test (lower(s)); + CREATE TRIGGER t BEFORE UPDATE ON hot_expr_test + FOR EACH ROW EXECUTE FUNCTION hot_expr_trig(); + INSERT INTO hot_expr_test SELECT g, 'abc' || g, 0 FROM generate_series(1, 10) g$$, + $$UPDATE hot_expr_test SET x = 1$$); +DROP FUNCTION hot_expr_trig() CASCADE; + +-- Partial index, both versions outside the predicate, changing the key or +-- the predicate's column. +SELECT * FROM update_case( + $$CREATE TABLE hot_expr_test (id int PRIMARY KEY, status text, v int) WITH (fillfactor = 50); + CREATE INDEX ON hot_expr_test (v) WHERE status = 'active'; + INSERT INTO hot_expr_test SELECT g, 'done', g FROM generate_series(1, 10) g$$, + $$UPDATE hot_expr_test SET v = v + 1$$); + +SELECT * FROM update_case( + $$CREATE TABLE hot_expr_test (id int PRIMARY KEY, status text, v int) WITH (fillfactor = 50); + CREATE INDEX ON hot_expr_test (v) WHERE status = 'active'; + INSERT INTO hot_expr_test SELECT g, 'done', g FROM generate_series(1, 10) g$$, + $$UPDATE hot_expr_test SET status = 'archived'$$); + +-- Both versions inside the predicate with the key unchanged. +SELECT * FROM update_case( + $$CREATE TABLE hot_expr_test (id int PRIMARY KEY, status text, v int) WITH (fillfactor = 50); + CREATE INDEX ON hot_expr_test (v) WHERE status <> 'x'; + INSERT INTO hot_expr_test SELECT g, 'a', g FROM generate_series(1, 10) g$$, + $$UPDATE hot_expr_test SET status = 'b'$$); + +-- Entering or leaving the predicate, or changing the key inside it. +SELECT * FROM update_case( + $$CREATE TABLE hot_expr_test (id int PRIMARY KEY, status text, v int) WITH (fillfactor = 50); + CREATE INDEX ON hot_expr_test (v) WHERE status = 'active'; + INSERT INTO hot_expr_test SELECT g, 'active', g FROM generate_series(1, 10) g$$, + $$UPDATE hot_expr_test SET status = 'done'$$); + +SELECT * FROM update_case( + $$CREATE TABLE hot_expr_test (id int PRIMARY KEY, status text, v int) WITH (fillfactor = 50); + CREATE INDEX ON hot_expr_test (v) WHERE status = 'active'; + INSERT INTO hot_expr_test SELECT g, 'done', g FROM generate_series(1, 10) g$$, + $$UPDATE hot_expr_test SET status = 'active'$$); + +SELECT * FROM update_case( + $$CREATE TABLE hot_expr_test (id int PRIMARY KEY, status text, v int) WITH (fillfactor = 50); + CREATE INDEX ON hot_expr_test (v) WHERE status = 'active'; + INSERT INTO hot_expr_test SELECT g, 'active', g FROM generate_series(1, 10) g$$, + $$UPDATE hot_expr_test SET v = v + 1000$$); + +-- Rows updated outside a partial index must not be found through it. +SELECT * FROM update_case( + $$CREATE TABLE hot_expr_test (id int PRIMARY KEY, status text, v int) WITH (fillfactor = 50); + CREATE INDEX hot_expr_test_v ON hot_expr_test (v) WHERE status = 'active'; + INSERT INTO hot_expr_test SELECT g, CASE WHEN g <= 5 THEN 'active' ELSE 'done' END, g + FROM generate_series(1, 10) g$$, + $$UPDATE hot_expr_test SET v = v + 100 WHERE status = 'done'$$); +SET enable_seqscan = off; +SET enable_bitmapscan = off; +EXPLAIN (COSTS OFF) SELECT id FROM hot_expr_test WHERE status = 'active' AND v > 0; +SELECT id FROM hot_expr_test WHERE status = 'active' AND v > 0 ORDER BY v; +RESET enable_seqscan; +RESET enable_bitmapscan; +SELECT id FROM hot_expr_test WHERE status = 'active' AND v > 0 ORDER BY v; + +DROP TABLE hot_expr_test; + +-- One UPDATE whose rows each need a different index to decide: some change +-- the expression index, some only enter or leave the partial index created +-- after it, some change neither. The partial index must hold exactly the +-- rows that satisfy its predicate. +CREATE TABLE hot_expr_multi (id int PRIMARY KEY, doc jsonb) WITH (fillfactor = 30); +CREATE INDEX hot_expr_multi_status ON hot_expr_multi ((doc->>'status')); +CREATE INDEX hot_expr_multi_vip ON hot_expr_multi ((doc->>'email')) + WHERE (doc->>'vip')::bool; +INSERT INTO hot_expr_multi + SELECT g, jsonb_build_object('status', 'a', 'email', 'e' || g, + 'vip', g % 2 = 0, 'n', 0) + FROM generate_series(1, 300) g; +UPDATE hot_expr_multi SET doc = CASE + WHEN id % 4 = 0 THEN jsonb_set(doc, '{status}', '"b"') + WHEN id % 4 = 1 THEN jsonb_set(doc, '{vip}', to_jsonb(NOT (doc->>'vip')::bool)) + WHEN id % 4 = 2 THEN jsonb_set(jsonb_set(doc, '{status}', '"c"'), + '{vip}', to_jsonb(NOT (doc->>'vip')::bool)) + ELSE jsonb_set(doc, '{n}', '1') END; +SET enable_seqscan = off; +SET enable_bitmapscan = off; +SELECT count(*) FROM hot_expr_multi + WHERE (doc->>'vip')::bool AND doc->>'email' >= 'e'; +RESET enable_seqscan; +RESET enable_bitmapscan; +SELECT count(*) FROM hot_expr_multi WHERE (doc->>'vip')::bool; +DROP TABLE hot_expr_multi; + +-- An index whose concurrent build failed is not valid, and its expression +-- need not be computable for every row. An UPDATE must not evaluate it. +CREATE TABLE hot_expr_cic (id int PRIMARY KEY, x int) WITH (fillfactor = 50); +INSERT INTO hot_expr_cic VALUES (1, 1), (2, 0); +CREATE INDEX CONCURRENTLY hot_expr_cic_idx ON hot_expr_cic ((100 / x)); +SELECT indisready, indisvalid FROM pg_index + WHERE indexrelid = 'hot_expr_cic_idx'::regclass; +UPDATE hot_expr_cic SET x = 0 WHERE id = 1; +UPDATE hot_expr_cic SET x = 1 WHERE id = 2; +SELECT * FROM hot_expr_cic ORDER BY id; +DROP TABLE hot_expr_cic; + +DROP FUNCTION update_case(text, text); + + +-- --------------------------------------------------------------------------- +-- Cleanup +-- --------------------------------------------------------------------------- +DROP FUNCTION has_hot_chain(text, tid); +DROP FUNCTION print_hot_chain(text, tid); +DROP FUNCTION get_hot_count(text); +-- Leave pageinspect installed: this runs inside the pageinspect regress suite +-- (page.sql created it), and other tests in the schedule still need it. diff --git a/src/test/isolation/expected/update-epq-indexes.out b/src/test/isolation/expected/update-epq-indexes.out new file mode 100644 index 00000000000..3a28bffa596 --- /dev/null +++ b/src/test/isolation/expected/update-epq-indexes.out @@ -0,0 +1,94 @@ +Parsed test spec with 2 sessions + +starting permutation: s2_begin s2_status s1_status s2_commit s1_check +step s2_begin: BEGIN; +step s2_status: UPDATE hot_epq SET status = 'b' WHERE id = 1; +step s1_status: UPDATE hot_epq SET status = 'a' WHERE id = 1; +step s2_commit: COMMIT; +step s1_status: <... completed> +step s1_check: + SET enable_seqscan = off; + SET enable_bitmapscan = off; + SELECT count(*) AS status_a FROM hot_epq WHERE status = 'a'; + SELECT count(*) AS doc_a FROM hot_epq WHERE doc->>'status' = 'a'; + SELECT count(*) AS vip FROM hot_epq WHERE vip AND id = 1; + RESET enable_seqscan; + RESET enable_bitmapscan; + +status_a +-------- + 1 +(1 row) + +doc_a +----- + 1 +(1 row) + +vip +--- + 1 +(1 row) + + +starting permutation: s2_begin s2_doc s1_doc s2_commit s1_check +step s2_begin: BEGIN; +step s2_doc: UPDATE hot_epq SET doc = jsonb_set(doc, ARRAY['status'], to_jsonb('b'::text)) WHERE id = 1; +step s1_doc: UPDATE hot_epq SET doc = jsonb_set(doc, ARRAY['status'], to_jsonb('a'::text)) WHERE id = 1; +step s2_commit: COMMIT; +step s1_doc: <... completed> +step s1_check: + SET enable_seqscan = off; + SET enable_bitmapscan = off; + SELECT count(*) AS status_a FROM hot_epq WHERE status = 'a'; + SELECT count(*) AS doc_a FROM hot_epq WHERE doc->>'status' = 'a'; + SELECT count(*) AS vip FROM hot_epq WHERE vip AND id = 1; + RESET enable_seqscan; + RESET enable_bitmapscan; + +status_a +-------- + 1 +(1 row) + +doc_a +----- + 1 +(1 row) + +vip +--- + 1 +(1 row) + + +starting permutation: s2_begin s2_vip s1_vip s2_commit s1_check +step s2_begin: BEGIN; +step s2_vip: UPDATE hot_epq SET vip = false WHERE id = 1; +step s1_vip: UPDATE hot_epq SET vip = true WHERE id = 1; +step s2_commit: COMMIT; +step s1_vip: <... completed> +step s1_check: + SET enable_seqscan = off; + SET enable_bitmapscan = off; + SELECT count(*) AS status_a FROM hot_epq WHERE status = 'a'; + SELECT count(*) AS doc_a FROM hot_epq WHERE doc->>'status' = 'a'; + SELECT count(*) AS vip FROM hot_epq WHERE vip AND id = 1; + RESET enable_seqscan; + RESET enable_bitmapscan; + +status_a +-------- + 1 +(1 row) + +doc_a +----- + 1 +(1 row) + +vip +--- + 1 +(1 row) + diff --git a/src/test/isolation/isolation_schedule b/src/test/isolation/isolation_schedule index bf9a2037c70..d7da899aa85 100644 --- a/src/test/isolation/isolation_schedule +++ b/src/test/isolation/isolation_schedule @@ -43,6 +43,7 @@ test: fk-fastpath-null-key test: subxid-overflow test: eval-plan-qual test: eval-plan-qual-trigger +test: update-epq-indexes test: eval-plan-qual-partition-prune test: inplace-inval test: intra-grant-inplace diff --git a/src/test/isolation/specs/update-epq-indexes.spec b/src/test/isolation/specs/update-epq-indexes.spec new file mode 100644 index 00000000000..09420c1a060 --- /dev/null +++ b/src/test/isolation/specs/update-epq-indexes.spec @@ -0,0 +1,55 @@ +# UPDATEs whose target row is updated concurrently, with a BEFORE ROW UPDATE +# trigger. The trigger code locks the latest version and rebuilds the new +# row from it, so deciding which indexes the UPDATE changes must compare the +# new row with that latest version, not with the one the UPDATE first read. +# +# The row starts with status a, inside the partial index. s2 changes it and +# holds the lock; s1 then sets it back and waits. Compared with the version +# s1 first read, s1 would change nothing, and every index would keep the +# entry for the version s2 wrote. + +setup +{ + CREATE TABLE hot_epq (id int PRIMARY KEY, status text, doc jsonb, vip bool) + WITH (fillfactor = 50); + CREATE INDEX hot_epq_status ON hot_epq (status); + CREATE INDEX hot_epq_doc ON hot_epq ((doc->>'status')); + CREATE INDEX hot_epq_vip ON hot_epq (id) WHERE vip; + INSERT INTO hot_epq VALUES (1, 'a', jsonb_build_object('status', 'a'), true); + CREATE FUNCTION hot_epq_trig() RETURNS trigger LANGUAGE plpgsql AS + $$BEGIN RETURN NEW; END$$; + CREATE TRIGGER hot_epq_before BEFORE UPDATE ON hot_epq + FOR EACH ROW EXECUTE FUNCTION hot_epq_trig(); +} + +teardown +{ + DROP TABLE hot_epq; + DROP FUNCTION hot_epq_trig(); +} + +session s1 +step s1_status { UPDATE hot_epq SET status = 'a' WHERE id = 1; } +step s1_doc { UPDATE hot_epq SET doc = jsonb_set(doc, ARRAY['status'], to_jsonb('a'::text)) WHERE id = 1; } +step s1_vip { UPDATE hot_epq SET vip = true WHERE id = 1; } +step s1_check +{ + SET enable_seqscan = off; + SET enable_bitmapscan = off; + SELECT count(*) AS status_a FROM hot_epq WHERE status = 'a'; + SELECT count(*) AS doc_a FROM hot_epq WHERE doc->>'status' = 'a'; + SELECT count(*) AS vip FROM hot_epq WHERE vip AND id = 1; + RESET enable_seqscan; + RESET enable_bitmapscan; +} + +session s2 +step s2_begin { BEGIN; } +step s2_status { UPDATE hot_epq SET status = 'b' WHERE id = 1; } +step s2_doc { UPDATE hot_epq SET doc = jsonb_set(doc, ARRAY['status'], to_jsonb('b'::text)) WHERE id = 1; } +step s2_vip { UPDATE hot_epq SET vip = false WHERE id = 1; } +step s2_commit { COMMIT; } + +permutation s2_begin s2_status s1_status s2_commit s1_check +permutation s2_begin s2_doc s1_doc s2_commit s1_check +permutation s2_begin s2_vip s1_vip s2_commit s1_check -- 2.50.1