From b5a6db686973e01eb0e43c98d3c8193764870850 Mon Sep 17 00:00:00 2001 From: Andrey Borodin Date: Sat, 24 Oct 2026 20:20:22 +0500 Subject: [PATCH v12 2/3] Expose physical WAL record sizes in pg_walinspect Whole-record compression makes a record's stored length differ from its decoded length. Reporting only decoded sizes hides the space saved by compression. Add physical lengths and totals alongside the existing decoded fields. Bump the extension version to 1.2, while allowing the C functions to serve the older SQL definitions until the extension is updated. Author: Andrey Borodin Discussion: https://www.postgresql.org/message-id/4DC38068-976E-4A84-8EE6-4EFACBBD927A@yandex-team.ru --- contrib/pg_walinspect/Makefile | 3 +- contrib/pg_walinspect/meson.build | 1 + .../pg_walinspect/pg_walinspect--1.1--1.2.sql | 103 ++++++++++++++++++ contrib/pg_walinspect/pg_walinspect.c | 32 ++++-- contrib/pg_walinspect/pg_walinspect.control | 2 +- doc/src/sgml/pgwalinspect.sgml | 13 +++ src/test/recovery/t/061_wal_compression.pl | 18 +++ 7 files changed, 161 insertions(+), 11 deletions(-) create mode 100644 contrib/pg_walinspect/pg_walinspect--1.1--1.2.sql diff --git a/contrib/pg_walinspect/Makefile b/contrib/pg_walinspect/Makefile index 22090f7716c..dc1c45ef7e9 100644 --- a/contrib/pg_walinspect/Makefile +++ b/contrib/pg_walinspect/Makefile @@ -7,7 +7,8 @@ OBJS = \ PGFILEDESC = "pg_walinspect - functions to inspect contents of PostgreSQL Write-Ahead Log" EXTENSION = pg_walinspect -DATA = pg_walinspect--1.0.sql pg_walinspect--1.0--1.1.sql +DATA = pg_walinspect--1.0.sql pg_walinspect--1.0--1.1.sql \ + pg_walinspect--1.1--1.2.sql REGRESS = pg_walinspect oldextversions diff --git a/contrib/pg_walinspect/meson.build b/contrib/pg_walinspect/meson.build index 82ab186c340..15c77285e0d 100644 --- a/contrib/pg_walinspect/meson.build +++ b/contrib/pg_walinspect/meson.build @@ -20,6 +20,7 @@ install_data( 'pg_walinspect.control', 'pg_walinspect--1.0.sql', 'pg_walinspect--1.0--1.1.sql', + 'pg_walinspect--1.1--1.2.sql', kwargs: contrib_data_args, ) diff --git a/contrib/pg_walinspect/pg_walinspect--1.1--1.2.sql b/contrib/pg_walinspect/pg_walinspect--1.1--1.2.sql new file mode 100644 index 00000000000..b5c1a9f345a --- /dev/null +++ b/contrib/pg_walinspect/pg_walinspect--1.1--1.2.sql @@ -0,0 +1,103 @@ +/* contrib/pg_walinspect/pg_walinspect--1.1--1.2.sql */ + +\echo Use "ALTER EXTENSION pg_walinspect UPDATE TO '1.2'" to load this file. \quit + +-- Append physical sizes without changing the existing, decoded-size columns. +DROP FUNCTION pg_get_wal_record_info(pg_lsn); +CREATE FUNCTION pg_get_wal_record_info(IN in_lsn pg_lsn, + OUT start_lsn pg_lsn, + OUT end_lsn pg_lsn, + OUT prev_lsn pg_lsn, + OUT xid xid, + OUT resource_manager text, + OUT record_type text, + OUT record_length int4, + OUT main_data_length int4, + OUT fpi_length int4, + OUT description text, + OUT block_ref text, + OUT physical_length int4 +) +AS 'MODULE_PATHNAME', 'pg_get_wal_record_info' +LANGUAGE C STRICT PARALLEL SAFE; + +REVOKE EXECUTE ON FUNCTION pg_get_wal_record_info(pg_lsn) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION pg_get_wal_record_info(pg_lsn) TO pg_read_server_files; + +DROP FUNCTION pg_get_wal_records_info(pg_lsn, pg_lsn); +CREATE FUNCTION pg_get_wal_records_info(IN start_lsn pg_lsn, + IN end_lsn pg_lsn, + OUT start_lsn pg_lsn, + OUT end_lsn pg_lsn, + OUT prev_lsn pg_lsn, + OUT xid xid, + OUT resource_manager text, + OUT record_type text, + OUT record_length int4, + OUT main_data_length int4, + OUT fpi_length int4, + OUT description text, + OUT block_ref text, + OUT physical_length int4 +) +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'pg_get_wal_records_info' +LANGUAGE C STRICT PARALLEL SAFE; + +REVOKE EXECUTE ON FUNCTION pg_get_wal_records_info(pg_lsn, pg_lsn) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION pg_get_wal_records_info(pg_lsn, pg_lsn) TO pg_read_server_files; + +DROP FUNCTION pg_get_wal_block_info(pg_lsn, pg_lsn, boolean); +CREATE FUNCTION pg_get_wal_block_info(IN start_lsn pg_lsn, + IN end_lsn pg_lsn, + IN show_data boolean DEFAULT true, + OUT start_lsn pg_lsn, + OUT end_lsn pg_lsn, + OUT prev_lsn pg_lsn, + OUT block_id int2, + OUT reltablespace oid, + OUT reldatabase oid, + OUT relfilenode oid, + OUT relforknumber int2, + OUT relblocknumber int8, + OUT xid xid, + OUT resource_manager text, + OUT record_type text, + OUT record_length int4, + OUT main_data_length int4, + OUT block_data_length int4, + OUT block_fpi_length int4, + OUT block_fpi_info text[], + OUT description text, + OUT block_data bytea, + OUT block_fpi_data bytea, + OUT physical_length int4 +) +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'pg_get_wal_block_info' +LANGUAGE C STRICT PARALLEL SAFE; + +REVOKE EXECUTE ON FUNCTION pg_get_wal_block_info(pg_lsn, pg_lsn, boolean) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION pg_get_wal_block_info(pg_lsn, pg_lsn, boolean) TO pg_read_server_files; + +DROP FUNCTION pg_get_wal_stats(pg_lsn, pg_lsn, boolean); +CREATE FUNCTION pg_get_wal_stats(IN start_lsn pg_lsn, + IN end_lsn pg_lsn, + IN per_record boolean DEFAULT false, + OUT "resource_manager/record_type" text, + OUT count int8, + OUT count_percentage float8, + OUT record_size int8, + OUT record_size_percentage float8, + OUT fpi_size int8, + OUT fpi_size_percentage float8, + OUT combined_size int8, + OUT combined_size_percentage float8, + OUT physical_size int8 +) +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'pg_get_wal_stats' +LANGUAGE C STRICT PARALLEL SAFE; + +REVOKE EXECUTE ON FUNCTION pg_get_wal_stats(pg_lsn, pg_lsn, boolean) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION pg_get_wal_stats(pg_lsn, pg_lsn, boolean) TO pg_read_server_files; diff --git a/contrib/pg_walinspect/pg_walinspect.c b/contrib/pg_walinspect/pg_walinspect.c index a172f9e2b40..97ffb17d59a 100644 --- a/contrib/pg_walinspect/pg_walinspect.c +++ b/contrib/pg_walinspect/pg_walinspect.c @@ -60,6 +60,7 @@ static void FillXLogStatsRow(const char *name, uint64 n, uint64 total_count, uint64 rec_len, uint64 total_rec_len, uint64 fpi_len, uint64 total_fpi_len, uint64 tot_len, uint64 total_len, + uint64 physical_len, Datum *values, bool *nulls, uint32 ncols); static void GetWalStats(FunctionCallInfo fcinfo, XLogRecPtr start_lsn, @@ -244,6 +245,10 @@ GetWALRecordInfo(XLogReaderState *record, Datum *values, else nulls[i++] = true; + /* The physical length was added in extension version 1.2. */ + if (ncols > i) + values[i++] = Int32GetDatum(XLogRecGetPhysicalLen(record)); + Assert(i == ncols); } @@ -262,7 +267,7 @@ static void GetWALBlockInfo(FunctionCallInfo fcinfo, XLogReaderState *record, bool show_data) { -#define PG_GET_WAL_BLOCK_INFO_COLS 20 +#define PG_GET_WAL_BLOCK_INFO_COLS 21 int block_id; ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; RmgrData desc; @@ -408,7 +413,10 @@ GetWALBlockInfo(FunctionCallInfo fcinfo, XLogReaderState *record, else nulls[i++] = true; - Assert(i == PG_GET_WAL_BLOCK_INFO_COLS); + if (rsinfo->setDesc->natts > i) + values[i++] = Int32GetDatum(XLogRecGetPhysicalLen(record)); + + Assert(i == rsinfo->setDesc->natts); /* Store a tuple for this block reference */ tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, @@ -472,7 +480,7 @@ pg_get_wal_block_info(PG_FUNCTION_ARGS) Datum pg_get_wal_record_info(PG_FUNCTION_ARGS) { -#define PG_GET_WAL_RECORD_INFO_COLS 11 +#define PG_GET_WAL_RECORD_INFO_COLS 12 Datum result; Datum values[PG_GET_WAL_RECORD_INFO_COLS] = {0}; bool nulls[PG_GET_WAL_RECORD_INFO_COLS] = {0}; @@ -504,7 +512,7 @@ pg_get_wal_record_info(PG_FUNCTION_ARGS) errmsg("could not read WAL at %X/%08X", LSN_FORMAT_ARGS(xlogreader->EndRecPtr)))); - GetWALRecordInfo(xlogreader, values, nulls, PG_GET_WAL_RECORD_INFO_COLS); + GetWALRecordInfo(xlogreader, values, nulls, tupdesc->natts); pfree(xlogreader->private_data); XLogReaderFree(xlogreader); @@ -550,7 +558,7 @@ static void GetWALRecordsInfo(FunctionCallInfo fcinfo, XLogRecPtr start_lsn, XLogRecPtr end_lsn) { -#define PG_GET_WAL_RECORDS_INFO_COLS 11 +#define PG_GET_WAL_RECORDS_INFO_COLS 12 XLogReaderState *xlogreader; ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; MemoryContext old_cxt; @@ -576,7 +584,7 @@ GetWALRecordsInfo(FunctionCallInfo fcinfo, XLogRecPtr start_lsn, old_cxt = MemoryContextSwitchTo(tmp_cxt); GetWALRecordInfo(xlogreader, values, nulls, - PG_GET_WAL_RECORDS_INFO_COLS); + rsinfo->setDesc->natts); tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, values, nulls); @@ -619,6 +627,7 @@ FillXLogStatsRow(const char *name, uint64 rec_len, uint64 total_rec_len, uint64 fpi_len, uint64 total_fpi_len, uint64 tot_len, uint64 total_len, + uint64 physical_len, Datum *values, bool *nulls, uint32 ncols) { double n_pct, @@ -653,6 +662,9 @@ FillXLogStatsRow(const char *name, values[i++] = Int64GetDatum(tot_len); values[i++] = Float8GetDatum(tot_len_pct); + if (ncols > i) + values[i++] = Int64GetDatum(physical_len); + Assert(i == ncols); } @@ -734,6 +746,7 @@ GetXLogSummaryStats(XLogStats *stats, ReturnSetInfo *rsinfo, FillXLogStatsRow(psprintf("%s/%s", desc.rm_name, id), count, total_count, rec_len, total_rec_len, fpi_len, total_fpi_len, tot_len, total_len, + stats->record_stats[ri][rj].physical_len, values, nulls, ncols); tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, @@ -755,7 +768,8 @@ GetXLogSummaryStats(XLogStats *stats, ReturnSetInfo *rsinfo, FillXLogStatsRow(desc.rm_name, count, total_count, rec_len, total_rec_len, fpi_len, total_fpi_len, tot_len, - total_len, values, nulls, ncols); + total_len, stats->rmgr_stats[ri].physical_len, + values, nulls, ncols); tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, values, nulls); @@ -776,7 +790,7 @@ static void GetWalStats(FunctionCallInfo fcinfo, XLogRecPtr start_lsn, XLogRecPtr end_lsn, bool stats_per_record) { -#define PG_GET_WAL_STATS_COLS 9 +#define PG_GET_WAL_STATS_COLS 10 XLogReaderState *xlogreader; XLogStats stats = {0}; ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; @@ -801,7 +815,7 @@ GetWalStats(FunctionCallInfo fcinfo, XLogRecPtr start_lsn, XLogRecPtr end_lsn, XLogReaderFree(xlogreader); GetXLogSummaryStats(&stats, rsinfo, values, nulls, - PG_GET_WAL_STATS_COLS, + rsinfo->setDesc->natts, stats_per_record); #undef PG_GET_WAL_STATS_COLS diff --git a/contrib/pg_walinspect/pg_walinspect.control b/contrib/pg_walinspect/pg_walinspect.control index efa3cb2cfe5..5f574b865bb 100644 --- a/contrib/pg_walinspect/pg_walinspect.control +++ b/contrib/pg_walinspect/pg_walinspect.control @@ -1,5 +1,5 @@ # pg_walinspect extension comment = 'functions to inspect contents of PostgreSQL Write-Ahead Log' -default_version = '1.1' +default_version = '1.2' module_pathname = '$libdir/pg_walinspect' relocatable = true diff --git a/doc/src/sgml/pgwalinspect.sgml b/doc/src/sgml/pgwalinspect.sgml index 79c3ead40bc..bf072d3786c 100644 --- a/doc/src/sgml/pgwalinspect.sgml +++ b/doc/src/sgml/pgwalinspect.sgml @@ -59,6 +59,19 @@ General Functions + + Record lengths and the record_size, + fpi_size, and combined_size + statistics describe records before whole-record compression, but after any + per-image compression. physical_length and + physical_size, added in version 1.2, give the + corresponding sizes stored in WAL, including compression headers but not + WAL page headers or alignment padding. Divide physical size by combined + size to measure the whole-record compression ratio. The physical payload + of a compressed record cannot be apportioned between its images and other + data. + + diff --git a/src/test/recovery/t/061_wal_compression.pl b/src/test/recovery/t/061_wal_compression.pl index c7f7cf5302d..584d6c06a9e 100644 --- a/src/test/recovery/t/061_wal_compression.pl +++ b/src/test/recovery/t/061_wal_compression.pl @@ -172,6 +172,24 @@ sub test_wal_compression my $end_lsn = $primary->safe_psql('postgres', 'SELECT pg_current_wal_flush_lsn()'); + is($primary->safe_psql('postgres', qq{ + WITH records AS ( + SELECT sum(record_length) AS decoded, sum(physical_length) AS physical + FROM pg_get_wal_records_info('$start_lsn', '$end_lsn')), + stats AS ( + SELECT sum(combined_size) AS decoded, sum(physical_size) AS physical + FROM pg_get_wal_stats('$start_lsn', '$end_lsn')) + SELECT records.decoded = stats.decoded AND records.physical = stats.physical + FROM records, stats; + }), 't', "physical and decoded statistics agree ($method)"); + + my $sizes_differ = $primary->safe_psql('postgres', qq{ + SELECT bool_or(physical_length < record_length) + FROM pg_get_wal_records_info('$start_lsn', '$end_lsn'); + }); + is($sizes_differ, $method eq 'pglz' ? 'f' : 't', + "physical sizes reflect whole-record compression ($method)"); + $primary->command_ok( [ 'pg_waldump', -- That's all, folks. May the source be with you.