diff --git a/contrib/pg_xlogdump/pg_xlogdump.c b/contrib/pg_xlogdump/pg_xlogdump.c index c555786..239321f 100644 --- a/contrib/pg_xlogdump/pg_xlogdump.c +++ b/contrib/pg_xlogdump/pg_xlogdump.c @@ -15,12 +15,28 @@ #include #include +#include "access/clog.h" +#include "access/gin_private.h" +#include "access/gist_private.h" +#include "access/heapam_xlog.h" +#include "access/heapam_xlog.h" +#include "access/multixact.h" +#include "access/nbtree.h" +#include "access/spgist_private.h" +#include "access/xact.h" #include "access/xlog.h" #include "access/xlogreader.h" #include "access/transam.h" +#include "catalog/pg_control.h" +#include "catalog/storage_xlog.h" +#include "commands/dbcommands.h" +#include "commands/sequence.h" +#include "commands/tablespace.h" #include "common/fe_memutils.h" #include "getopt_long.h" #include "rmgrdesc.h" +#include "storage/standby.h" +#include "utils/relmapper.h" static const char *progname; @@ -41,6 +57,8 @@ typedef struct XLogDumpConfig int stop_after_records; int already_displayed_records; bool follow; + bool stats; + bool stats_per_record; /* filter options */ int filter_by_rmgr; @@ -48,6 +66,20 @@ typedef struct XLogDumpConfig bool filter_by_xid_enabled; } XLogDumpConfig; +typedef struct Stats +{ + uint64 count; + uint64 rec_len; + uint64 fpi_len; +} Stats; + +typedef struct XLogDumpStats +{ + uint64 count; + Stats rmgr_stats[RM_NEXT_ID]; + Stats record_stats[RM_NEXT_ID][16]; +} XLogDumpStats; + static void fatal_error(const char *fmt,...) __attribute__((format(PG_PRINTF_ATTRIBUTE, 1, 2))); @@ -322,6 +354,50 @@ XLogDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen, } /* + * Store per-rmgr and per-record statistics for a given record. + */ +static void +XLogDumpCountRecord(XLogDumpConfig *config, XLogDumpStats *stats, XLogRecPtr ReadRecPtr, XLogRecord *record) +{ + RmgrId rmid; + uint8 recid; + + if (config->filter_by_rmgr != -1 && + config->filter_by_rmgr != record->xl_rmid) + return; + + if (config->filter_by_xid_enabled && + config->filter_by_xid != record->xl_xid) + return; + + stats->count++; + + /* Update per-rmgr statistics */ + + rmid = record->xl_rmid; + + stats->rmgr_stats[rmid].count++; + stats->rmgr_stats[rmid].rec_len += + record->xl_len + SizeOfXLogRecord; + stats->rmgr_stats[rmid].fpi_len += + record->xl_tot_len - (record->xl_len + SizeOfXLogRecord); + + /* + * Update per-record statistics, where the record is identified by a + * combination of the RmgrId and the upper four bits of the xl_info + * field (to give sixteen possible entries per RmgrId). + */ + + recid = (record->xl_info & ~XLR_INFO_MASK) >> 4; + + stats->record_stats[rmid][recid].count++; + stats->record_stats[rmid][recid].rec_len += + record->xl_len + SizeOfXLogRecord; + stats->record_stats[rmid][recid].fpi_len += + record->xl_tot_len - (record->xl_len + SizeOfXLogRecord); +} + +/* * Print a record to stdout */ static void @@ -380,6 +456,498 @@ XLogDumpDisplayRecord(XLogDumpConfig *config, XLogRecPtr ReadRecPtr, XLogRecord } } +/* + * Given an xl_rmid and the high bits of xl_info, returns a string + * describing the record type. + */ +static const char * +identify_record(RmgrId rmid, uint8 info) +{ + const RmgrDescData *desc = &RmgrDescTable[rmid]; + const char *rec; + + rec = psprintf("0x%x", info); + + switch (rmid) + { + case RM_XLOG_ID: + switch (info) + { + case XLOG_CHECKPOINT_SHUTDOWN: + rec = "CHECKPOINT_SHUTDOWN"; + break; + case XLOG_CHECKPOINT_ONLINE: + rec = "CHECKPOINT_ONLINE"; + break; + case XLOG_NOOP: + rec = "NOOP"; + break; + case XLOG_NEXTOID: + rec = "NEXTOID"; + break; + case XLOG_SWITCH: + rec = "SWITCH"; + break; + case XLOG_BACKUP_END: + rec = "BACKUP_END"; + break; + case XLOG_PARAMETER_CHANGE: + rec = "PARAMETER_CHANGE"; + break; + case XLOG_RESTORE_POINT: + rec = "RESTORE_POINT"; + break; + case XLOG_FPW_CHANGE: + rec = "FPW_CHANGE"; + break; + case XLOG_END_OF_RECOVERY: + rec = "END_OF_RECOVERY"; + break; + case XLOG_FPI: + rec = "FPI"; + break; + } + break; + + case RM_XACT_ID: + switch (info) + { + case XLOG_XACT_COMMIT: + rec = "COMMIT"; + break; + case XLOG_XACT_PREPARE: + rec = "PREPARE"; + break; + case XLOG_XACT_ABORT: + rec = "ABORT"; + break; + case XLOG_XACT_COMMIT_PREPARED: + rec = "COMMIT_PREPARED"; + break; + case XLOG_XACT_ABORT_PREPARED: + rec = "ABORT_PREPARED"; + break; + case XLOG_XACT_ASSIGNMENT: + rec = "ASSIGNMENT"; + break; + case XLOG_XACT_COMMIT_COMPACT: + rec = "COMMIT_COMPACT"; + break; + } + break; + + case RM_SMGR_ID: + switch (info) + { + case XLOG_SMGR_CREATE: + rec = "CREATE"; + break; + case XLOG_SMGR_TRUNCATE: + rec = "TRUNCATE"; + break; + } + break; + + case RM_CLOG_ID: + switch (info) + { + case CLOG_ZEROPAGE: + rec = "ZEROPAGE"; + break; + case CLOG_TRUNCATE: + rec = "TRUNCATE"; + break; + } + break; + + case RM_DBASE_ID: + switch (info) + { + case XLOG_DBASE_CREATE: + rec = "CREATE"; + break; + case XLOG_DBASE_DROP: + rec = "DROP"; + break; + } + break; + + case RM_TBLSPC_ID: + switch (info) + { + case XLOG_TBLSPC_CREATE: + rec = "CREATE"; + break; + case XLOG_TBLSPC_DROP: + rec = "DROP"; + break; + } + break; + + case RM_MULTIXACT_ID: + switch (info) + { + case XLOG_MULTIXACT_ZERO_OFF_PAGE: + rec = "ZERO_OFF_PAGE"; + break; + case XLOG_MULTIXACT_ZERO_MEM_PAGE: + rec = "ZERO_MEM_PAGE"; + break; + case XLOG_MULTIXACT_CREATE_ID: + rec = "CREATE_ID"; + break; + } + break; + + case RM_RELMAP_ID: + switch (info) + { + case XLOG_RELMAP_UPDATE: + rec = "UPDATE"; + break; + } + break; + + case RM_STANDBY_ID: + switch (info) + { + case XLOG_STANDBY_LOCK: + rec = "LOCK"; + break; + case XLOG_RUNNING_XACTS: + rec = "RUNNING_XACTS"; + break; + } + break; + + case RM_HEAP2_ID: + switch (info & XLOG_HEAP_OPMASK) + { + case XLOG_HEAP2_CLEAN: + rec = "CLEAN"; + break; + case XLOG_HEAP2_FREEZE_PAGE: + rec = "FREEZE_PAGE"; + break; + case XLOG_HEAP2_CLEANUP_INFO: + rec = "CLEANUP_INFO"; + break; + case XLOG_HEAP2_VISIBLE: + rec = "VISIBLE"; + break; + case XLOG_HEAP2_MULTI_INSERT: + rec = "MULTI_INSERT"; + break; + case XLOG_HEAP2_LOCK_UPDATED: + rec = "LOCK_UPDATED"; + break; + case XLOG_HEAP2_NEW_CID: + rec = "NEW_CID"; + break; + case XLOG_HEAP2_REWRITE: + rec = "REWRITE"; + break; + } + + if (info & XLOG_HEAP_INIT_PAGE) + rec = psprintf("%s+INIT", rec); + + break; + + case RM_HEAP_ID: + switch (info & XLOG_HEAP_OPMASK) + { + case XLOG_HEAP_INSERT: + rec = "INSERT"; + break; + case XLOG_HEAP_DELETE: + rec = "DELETE"; + break; + case XLOG_HEAP_UPDATE: + rec = "UPDATE"; + break; + case XLOG_HEAP_HOT_UPDATE: + rec = "HOT_UPDATE"; + break; + case XLOG_HEAP_NEWPAGE: + rec = "NEWPAGE"; + break; + case XLOG_HEAP_LOCK: + rec = "LOCK"; + break; + case XLOG_HEAP_INPLACE: + rec = "INPLACE"; + break; + } + + if (info & XLOG_HEAP_INIT_PAGE) + rec = psprintf("%s+INIT", rec); + + break; + + case RM_BTREE_ID: + switch (info) + { + case XLOG_BTREE_INSERT_LEAF: + rec = "INSERT_LEAF"; + break; + case XLOG_BTREE_INSERT_UPPER: + rec = "INSERT_UPPER"; + break; + case XLOG_BTREE_INSERT_META: + rec = "INSERT_META"; + break; + case XLOG_BTREE_SPLIT_L: + rec = "SPLIT_L"; + break; + case XLOG_BTREE_SPLIT_R: + rec = "SPLIT_R"; + break; + case XLOG_BTREE_SPLIT_L_ROOT: + rec = "SPLIT_L_ROOT"; + break; + case XLOG_BTREE_SPLIT_R_ROOT: + rec = "SPLIT_R_ROOT"; + break; + case XLOG_BTREE_VACUUM: + rec = "VACUUM"; + break; + case XLOG_BTREE_DELETE: + rec = "DELETE"; + break; + case XLOG_BTREE_MARK_PAGE_HALFDEAD: + rec = "MARK_PAGE_HALFDEAD"; + break; + case XLOG_BTREE_UNLINK_PAGE: + rec = "UNLINK_PAGE"; + break; + case XLOG_BTREE_UNLINK_PAGE_META: + rec = "UNLINK_PAGE_META"; + break; + case XLOG_BTREE_NEWROOT: + rec = "NEWROOT"; + break; + case XLOG_BTREE_REUSE_PAGE: + rec = "REUSE_PAGE"; + break; + } + break; + + case RM_HASH_ID: + break; + + case RM_GIN_ID: + switch (info) + { + case XLOG_GIN_CREATE_INDEX: + rec = "CREATE_INDEX"; + break; + case XLOG_GIN_CREATE_PTREE: + rec = "CREATE_PTREE"; + break; + case XLOG_GIN_INSERT: + rec = "INSERT"; + break; + case XLOG_GIN_SPLIT: + rec = "SPLIT"; + break; + case XLOG_GIN_VACUUM_PAGE: + rec = "VACUUM_PAGE"; + break; + case XLOG_GIN_VACUUM_DATA_LEAF_PAGE: + rec = "VACUUM_DATA_LEAF_PAGE"; + break; + case XLOG_GIN_DELETE_PAGE: + rec = "DELETE_PAGE"; + break; + case XLOG_GIN_UPDATE_META_PAGE: + rec = "UPDATE_META_PAGE"; + break; + case XLOG_GIN_INSERT_LISTPAGE: + rec = "INSERT_LISTPAGE"; + break; + case XLOG_GIN_DELETE_LISTPAGE: + rec = "DELETE_LISTPAGE"; + break; + } + break; + + case RM_GIST_ID: + switch (info) + { + case XLOG_GIST_PAGE_UPDATE: + rec = "PAGE_UPDATE"; + break; + case XLOG_GIST_PAGE_SPLIT: + rec = "PAGE_SPLIT"; + break; + case XLOG_GIST_CREATE_INDEX: + rec = "CREATE_INDEX"; + break; + } + break; + + case RM_SEQ_ID: + switch (info) + { + case XLOG_SEQ_LOG: + rec = "LOG"; + break; + } + break; + + case RM_SPGIST_ID: + switch (info) + { + case XLOG_SPGIST_CREATE_INDEX: + rec = "CREATE_INDEX"; + break; + case XLOG_SPGIST_ADD_LEAF: + rec = "ADD_LEAF"; + break; + case XLOG_SPGIST_MOVE_LEAFS: + rec = "MOVE_LEAFS"; + break; + case XLOG_SPGIST_ADD_NODE: + rec = "ADD_NODE"; + break; + case XLOG_SPGIST_SPLIT_TUPLE: + rec = "SPLIT_TUPLE"; + break; + case XLOG_SPGIST_PICKSPLIT: + rec = "PICKSPLIT"; + break; + case XLOG_SPGIST_VACUUM_LEAF: + rec = "VACUUM_LEAF"; + break; + case XLOG_SPGIST_VACUUM_ROOT: + rec = "VACUUM_ROOT"; + break; + case XLOG_SPGIST_VACUUM_REDIRECT: + rec = "VACUUM_REDIRECT"; + break; + } + break; + + default: + break; + } + + return psprintf("%s/%s", desc->rm_name, rec); +} + +/* + * Display a single row of record counts and sizes for an rmgr or record. + */ +static void +XLogDumpStatsRow(const char *name, + uint64 n, double n_pct, + uint64 rec_len, double rec_len_pct, + uint64 fpi_len, double fpi_len_pct, + uint64 total_len, double total_len_pct) +{ + printf("%-27s %20zu (%6.02f) %20zu (%6.02f) %20zu (%6.02f) %20zu (%6.02f)\n", + name, n, n_pct, rec_len, rec_len_pct, fpi_len, fpi_len_pct, + total_len, total_len_pct); +} + + +/* + * Display summary statistics about the records seen so far. + */ +static void +XLogDumpDisplayStats(XLogDumpConfig *config, XLogDumpStats *stats) +{ + int ri, rj; + uint64 total_count = 0; + uint64 total_rec_len = 0; + uint64 total_fpi_len = 0; + uint64 total_len = 0; + + /* + * Make a first pass to calculate column totals: + * count(*), + * sum(xl_len+SizeOfXLogRecord), + * sum(xl_tot_len-xl_len-SizeOfXLogRecord), and + * sum(xl_tot_len). + * These are used to calculate percentages for each record type. + */ + + for (ri = 0; ri < RM_NEXT_ID; ri++) + { + total_count += stats->rmgr_stats[ri].count; + total_rec_len += stats->rmgr_stats[ri].rec_len; + total_fpi_len += stats->rmgr_stats[ri].fpi_len; + } + total_len = total_rec_len+total_fpi_len; + + /* + * 27 is strlen("Transaction/COMMIT_PREPARED"), + * 20 is strlen(2^64), 8 is strlen("(100.00%)") + */ + + printf("%-27s %20s %8s %20s %8s %20s %8s %20s %8s\n" + "%-27s %20s %8s %20s %8s %20s %8s %20s %8s\n", + "Type", "N", "(%)", "Record size", "(%)", "FPI size", "(%)", "Combined size", "(%)", + "----", "-", "---", "-----------", "---", "--------", "---", "-------------", "---"); + + for (ri = 0; ri < RM_NEXT_ID; ri++) + { + uint64 count, rec_len, fpi_len; + + if (!config->stats_per_record) + { + const RmgrDescData *desc = &RmgrDescTable[ri]; + + count = stats->rmgr_stats[ri].count; + rec_len = stats->rmgr_stats[ri].rec_len; + fpi_len = stats->rmgr_stats[ri].fpi_len; + + XLogDumpStatsRow(desc->rm_name, + count, 100*(double)count/total_count, + rec_len, 100*(double)rec_len/total_rec_len, + fpi_len, 100*(double)fpi_len/total_fpi_len, + rec_len+fpi_len, 100*(double)(rec_len+fpi_len)/total_len); + } + else + { + for (rj = 0; rj < 16; rj++) + { + count = stats->record_stats[ri][rj].count; + rec_len = stats->record_stats[ri][rj].rec_len; + fpi_len = stats->record_stats[ri][rj].fpi_len; + + /* Skip undefined combinations and ones that didn't occur */ + if (count == 0) + continue; + + XLogDumpStatsRow(identify_record(ri, rj << 4), + count, 100*(double)count/total_count, + rec_len, 100*(double)rec_len/total_rec_len, + fpi_len, 100*(double)fpi_len/total_fpi_len, + rec_len+fpi_len, 100*(double)(rec_len+fpi_len)/total_len); + } + } + } + + printf("%-27s %20s %8s %20s %8s %20s %8s %20s\n", + "", "--------", "", "--------", "", "--------", "", "--------"); + + /* + * The percentages in earlier rows were calculated against the + * column total, but the ones that follow are against the row total. + * Note that these are displayed with a % symbol to differentiate + * them from the earlier ones, and are thus up to 9 characters long. + */ + + printf("%-27s %20zu %-9s%20zu %-9s%20zu %-9s%20zu %-6s\n", + "Total", + stats->count, "", + total_rec_len, psprintf("[%.02f%%]", 100*(double)total_rec_len/total_len), + total_fpi_len, psprintf("[%.02f%%]", 100*(double)total_fpi_len/total_len), + total_len, "[100%]"); +} + static void usage(void) { @@ -401,6 +969,8 @@ usage(void) printf(" (default: 1 or the value used in STARTSEG)\n"); printf(" -V, --version output version information, then exit\n"); printf(" -x, --xid=XID only show records with TransactionId XID\n"); + printf(" -z, --stats show per-rmgr statistics\n"); + printf(" -Z, --record-stats show per-record statistics\n"); printf(" -?, --help show this help, then exit\n"); } @@ -412,6 +982,7 @@ main(int argc, char **argv) XLogReaderState *xlogreader_state; XLogDumpPrivate private; XLogDumpConfig config; + XLogDumpStats stats; XLogRecord *record; XLogRecPtr first_record; char *errormsg; @@ -428,6 +999,8 @@ main(int argc, char **argv) {"timeline", required_argument, NULL, 't'}, {"xid", required_argument, NULL, 'x'}, {"version", no_argument, NULL, 'V'}, + {"stats", no_argument, NULL, 'z'}, + {"record-stats", no_argument, NULL, 'Z'}, {NULL, 0, NULL, 0} }; @@ -438,6 +1011,7 @@ main(int argc, char **argv) memset(&private, 0, sizeof(XLogDumpPrivate)); memset(&config, 0, sizeof(XLogDumpConfig)); + memset(&stats, 0, sizeof(XLogDumpStats)); private.timeline = 1; private.startptr = InvalidXLogRecPtr; @@ -451,6 +1025,8 @@ main(int argc, char **argv) config.filter_by_rmgr = -1; config.filter_by_xid = InvalidTransactionId; config.filter_by_xid_enabled = false; + config.stats = false; + config.stats_per_record = false; if (argc <= 1) { @@ -458,7 +1034,7 @@ main(int argc, char **argv) goto bad_argument; } - while ((option = getopt_long(argc, argv, "be:?fn:p:r:s:t:Vx:", + while ((option = getopt_long(argc, argv, "be:?fn:p:r:s:t:Vx:zZ", long_options, &optindex)) != -1) { switch (option) @@ -551,6 +1127,13 @@ main(int argc, char **argv) } config.filter_by_xid_enabled = true; break; + case 'z': + config.stats = true; + break; + case 'Z': + config.stats = true; + config.stats_per_record = true; + break; default: goto bad_argument; } @@ -711,7 +1294,10 @@ main(int argc, char **argv) /* after reading the first record, continue at next one */ first_record = InvalidXLogRecPtr; - XLogDumpDisplayRecord(&config, xlogreader_state->ReadRecPtr, record); + if (config.stats == true) + XLogDumpCountRecord(&config, &stats, xlogreader_state->ReadRecPtr, record); + else + XLogDumpDisplayRecord(&config, xlogreader_state->ReadRecPtr, record); /* check whether we printed enough */ if (config.stop_after_records > 0 && @@ -719,6 +1305,9 @@ main(int argc, char **argv) break; } + if (config.stats == true) + XLogDumpDisplayStats(&config, &stats); + if (errormsg) fatal_error("error in WAL record at %X/%X: %s\n", (uint32) (xlogreader_state->ReadRecPtr >> 32), diff --git a/doc/src/sgml/pg_xlogdump.sgml b/doc/src/sgml/pg_xlogdump.sgml index 1d1a2ce..bfd9eb9 100644 --- a/doc/src/sgml/pg_xlogdump.sgml +++ b/doc/src/sgml/pg_xlogdump.sgml @@ -180,6 +180,28 @@ PostgreSQL documentation + + + + + Display summary statistics (number and size of records and + full-page images per rmgr) instead of individual records. + + + + + + + + + + Display summary statistics (number and size of records and + full-page images) instead of individual records. + + + + +