From d006044bdd6272fa4e37890b8e634b0b2a179dff Mon Sep 17 00:00:00 2001 From: Andrey Borodin Date: Sat, 27 Feb 2021 09:03:50 +0500 Subject: [PATCH v9 1/9] Allow alternate compression methods for wal_compression TODO: bump XLOG_PAGE_MAGIC --- doc/src/sgml/config.sgml | 9 +- doc/src/sgml/installation.sgml | 4 +- src/backend/Makefile | 2 +- src/backend/access/transam/xlog.c | 14 ++- src/backend/access/transam/xloginsert.c | 65 ++++++++++-- src/backend/access/transam/xlogreader.c | 99 ++++++++++++++++--- src/backend/utils/misc/guc.c | 21 ++-- src/backend/utils/misc/postgresql.conf.sample | 2 +- src/bin/pg_waldump/pg_waldump.c | 13 ++- src/include/access/xlog.h | 2 +- src/include/access/xlog_internal.h | 10 ++ src/include/access/xlogrecord.h | 15 ++- 12 files changed, 207 insertions(+), 49 deletions(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index aa3e178240..1df56d8034 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3127,23 +3127,26 @@ include_dir 'conf.d' - wal_compression (boolean) + wal_compression (enum) wal_compression configuration parameter - When this parameter is on, the PostgreSQL + This parameter enables compression of WAL using the specified + compression method. + When enabled, the PostgreSQL server compresses full page images written to WAL when is on or during a base backup. A compressed page image will be decompressed during WAL replay. + The supported methods are pglz and zlib. The default value is off. Only superusers can change this setting. - Turning this parameter on can reduce the WAL volume without + Enabling compression can reduce the WAL volume without increasing the risk of unrecoverable data corruption, but at the cost of some extra CPU spent on the compression during WAL logging and on the decompression during WAL replay. diff --git a/doc/src/sgml/installation.sgml b/doc/src/sgml/installation.sgml index 3c0aa118c7..073d5089f7 100644 --- a/doc/src/sgml/installation.sgml +++ b/doc/src/sgml/installation.sgml @@ -147,7 +147,7 @@ su - postgres specify the option to configure. Using this option disables support for compressed archives in pg_dump and - pg_restore. + pg_restore, and compressed WAL. @@ -1236,7 +1236,7 @@ build-postgresql: Prevents use of the Zlib library. This disables support for compressed archives in pg_dump - and pg_restore. + and pg_restore and compressed WAL. diff --git a/src/backend/Makefile b/src/backend/Makefile index 0da848b1fd..3af216ddfc 100644 --- a/src/backend/Makefile +++ b/src/backend/Makefile @@ -48,7 +48,7 @@ OBJS = \ LIBS := $(filter-out -lpgport -lpgcommon, $(LIBS)) $(LDAP_LIBS_BE) $(ICU_LIBS) # The backend doesn't need everything that's in LIBS, however -LIBS := $(filter-out -lz -lreadline -ledit -ltermcap -lncurses -lcurses, $(LIBS)) +LIBS := $(filter-out -lreadline -ledit -ltermcap -lncurses -lcurses, $(LIBS)) ifeq ($(with_systemd),yes) LIBS += -lsystemd diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 17eeff0720..1ccc51575a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -98,7 +98,7 @@ char *XLogArchiveCommand = NULL; bool EnableHotStandby = false; bool fullPageWrites = true; bool wal_log_hints = false; -bool wal_compression = false; +int wal_compression = WAL_COMPRESSION_NONE; char *wal_consistency_checking_string = NULL; bool *wal_consistency_checking = NULL; bool wal_init_zero = true; @@ -10470,7 +10470,17 @@ xlog_block_info(StringInfo buf, XLogReaderState *record) rnode.spcNode, rnode.dbNode, rnode.relNode, blk); if (XLogRecHasBlockImage(record, block_id)) - appendStringInfoString(buf, " FPW"); + { + int compression = + BKPIMAGE_IS_COMPRESSED(record->blocks[block_id].bimg_info) ? + BKPIMAGE_COMPRESSION(record->blocks[block_id].bimg_info) : -1; + if (compression == -1) + appendStringInfoString(buf, " FPW"); + else + appendStringInfo(buf, " FPW, compression method %d/%s", + compression, wal_compression_name(compression)); + } + } } diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 32b4cc84e7..4f81f19c49 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -33,8 +33,18 @@ #include "storage/proc.h" #include "utils/memutils.h" +#ifdef HAVE_LIBZ +#include +/* zlib compressBound is not a macro */ +#define ZLIB_MAX_BLCKSZ BLCKSZ + (BLCKSZ>>12) + (BLCKSZ>>14) + (BLCKSZ>>25) + 13 +#else +#define ZLIB_MAX_BLCKSZ 0 +#endif + /* Buffer size required to store a compressed version of backup block image */ -#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) +#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) + +#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ) /* * For each block reference registered with XLogRegisterBuffer, we fill in @@ -58,7 +68,7 @@ typedef struct * backup block data in XLogRecordAssemble() */ /* buffer to store a compressed version of backup block image */ - char compressed_page[PGLZ_MAX_BLCKSZ]; + char compressed_page[COMPRESS_BUFSIZE]; } registered_buffer; static registered_buffer *registered_buffers; @@ -113,7 +123,8 @@ static XLogRecData *XLogRecordAssemble(RmgrId rmid, uint8 info, XLogRecPtr RedoRecPtr, bool doPageWrites, XLogRecPtr *fpw_lsn, int *num_fpi); static bool XLogCompressBackupBlock(char *page, uint16 hole_offset, - uint16 hole_length, char *dest, uint16 *dlen); + uint16 hole_length, char *dest, + uint16 *dlen, WalCompression compression); /* * Begin constructing a WAL record. This must be called before the @@ -628,13 +639,14 @@ XLogRecordAssemble(RmgrId rmid, uint8 info, /* * Try to compress a block image if wal_compression is enabled */ - if (wal_compression) + if (wal_compression != WAL_COMPRESSION_NONE) { is_compressed = XLogCompressBackupBlock(page, bimg.hole_offset, cbimg.hole_length, regbuf->compressed_page, - &compressed_len); + &compressed_len, + wal_compression); } /* @@ -665,8 +677,13 @@ XLogRecordAssemble(RmgrId rmid, uint8 info, if (is_compressed) { + /* The current compression is stored in the WAL record */ + wal_compression_name(wal_compression); /* Range check */ + Assert(wal_compression < (1 << BKPIMAGE_COMPRESS_BITS)); + bimg.length = compressed_len; - bimg.bimg_info |= BKPIMAGE_IS_COMPRESSED; + bimg.bimg_info |= + wal_compression << BKPIMAGE_COMPRESS_OFFSET_BITS; rdt_datas_last->data = regbuf->compressed_page; rdt_datas_last->len = compressed_len; @@ -827,7 +844,7 @@ XLogRecordAssemble(RmgrId rmid, uint8 info, */ static bool XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, - char *dest, uint16 *dlen) + char *dest, uint16 *dlen, WalCompression compression) { int32 orig_len = BLCKSZ - hole_length; int32 len; @@ -853,12 +870,42 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, else source = page; + switch (compression) + { + case WAL_COMPRESSION_PGLZ: + len = pglz_compress(source, orig_len, dest, PGLZ_strategy_default); + break; + +#ifdef HAVE_LIBZ + case WAL_COMPRESSION_ZLIB: + { + unsigned long len_l = COMPRESS_BUFSIZE; + int ret; + ret = compress2((Bytef*)dest, &len_l, (Bytef*)source, orig_len, 1); + if (ret != Z_OK) + len_l = -1; + len = len_l; + break; + } +#endif + + default: + /* + * It should be impossible to get here for unsupported algorithms, + * which cannot be assigned if they're not enabled at compile time. + */ + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("unknown compression method requested: %d/%s", + compression, wal_compression_name(compression)))); + + } + /* - * We recheck the actual size even if pglz_compress() reports success and + * We recheck the actual size even if compression reports success and * see if the number of bytes saved by compression is larger than the * length of extra data needed for the compressed version of block image. */ - len = pglz_compress(source, orig_len, dest, PGLZ_strategy_default); if (len >= 0 && len + extra_bytes < orig_len) { diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 42738eb940..3a922caaeb 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -26,6 +26,7 @@ #include "catalog/pg_control.h" #include "common/pg_lzcompress.h" #include "replication/origin.h" +#include "utils/guc.h" #ifndef FRONTEND #include "miscadmin.h" @@ -33,6 +34,10 @@ #include "utils/memutils.h" #endif +#ifdef HAVE_LIBZ +#include +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -50,6 +55,29 @@ static void WALOpenSegmentInit(WALOpenSegment *seg, WALSegmentContext *segcxt, /* size of the buffer allocated for error message. */ #define MAX_ERRORMSG_LEN 1000 +/* + * Accept the likely variants for none and pglz, for compatibility with old + * server versions where wal_compression was a boolean. + */ +const struct config_enum_entry wal_compression_options[] = { + {"off", WAL_COMPRESSION_NONE, false}, + {"none", WAL_COMPRESSION_NONE, false}, + {"false", WAL_COMPRESSION_NONE, true}, + {"no", WAL_COMPRESSION_NONE, true}, + {"0", WAL_COMPRESSION_NONE, true}, + {"pglz", WAL_COMPRESSION_PGLZ, false}, + {"true", WAL_COMPRESSION_PGLZ, true}, + {"yes", WAL_COMPRESSION_PGLZ, true}, + {"on", WAL_COMPRESSION_PGLZ, true}, + {"1", WAL_COMPRESSION_PGLZ, true}, + +#ifdef HAVE_LIBZ + {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif + + {NULL, 0, false} +}; + /* * Construct a string in state->errormsg_buf explaining what's wrong with * the current record being read. @@ -1290,7 +1318,7 @@ DecodeXLogRecord(XLogReaderState *state, XLogRecord *record, char **errormsg) blk->apply_image = ((blk->bimg_info & BKPIMAGE_APPLY) != 0); - if (blk->bimg_info & BKPIMAGE_IS_COMPRESSED) + if (BKPIMAGE_IS_COMPRESSED(blk->bimg_info)) { if (blk->bimg_info & BKPIMAGE_HAS_HOLE) COPY_HEADER_FIELD(&blk->hole_length, sizeof(uint16)); @@ -1335,29 +1363,28 @@ DecodeXLogRecord(XLogReaderState *state, XLogRecord *record, char **errormsg) } /* - * cross-check that bimg_len < BLCKSZ if the IS_COMPRESSED - * flag is set. + * cross-check that bimg_len < BLCKSZ if it's compressed */ - if ((blk->bimg_info & BKPIMAGE_IS_COMPRESSED) && + if (BKPIMAGE_IS_COMPRESSED(blk->bimg_info) && blk->bimg_len == BLCKSZ) { report_invalid_record(state, - "BKPIMAGE_IS_COMPRESSED set, but block image length %u at %X/%X", + "BKPIMAGE_IS_COMPRESSED, but block image length %u at %X/%X", (unsigned int) blk->bimg_len, LSN_FORMAT_ARGS(state->ReadRecPtr)); goto err; } /* - * cross-check that bimg_len = BLCKSZ if neither HAS_HOLE nor - * IS_COMPRESSED flag is set. + * cross-check that bimg_len = BLCKSZ if neither HAS_HOLE is + * set nor IS_COMPRESSED(). */ if (!(blk->bimg_info & BKPIMAGE_HAS_HOLE) && - !(blk->bimg_info & BKPIMAGE_IS_COMPRESSED) && + !BKPIMAGE_IS_COMPRESSED(blk->bimg_info) && blk->bimg_len != BLCKSZ) { report_invalid_record(state, - "neither BKPIMAGE_HAS_HOLE nor BKPIMAGE_IS_COMPRESSED set, but block image length is %u at %X/%X", + "neither BKPIMAGE_HAS_HOLE nor BKPIMAGE_IS_COMPRESSED, but block image length is %u at %X/%X", (unsigned int) blk->data_len, LSN_FORMAT_ARGS(state->ReadRecPtr)); goto err; @@ -1535,6 +1562,22 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len) } } +/* + * Return a statically allocated string associated with the given compression + * method. + */ +const char * +wal_compression_name(WalCompression compression) +{ + for (int i=0; wal_compression_options[i].name != NULL; ++i) + { + if (wal_compression_options[i].val == compression) + return wal_compression_options[i].name; + } + + return "???"; +} + /* * Restore a full-page image from a backup block attached to an XLOG record. * @@ -1555,11 +1598,43 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) bkpb = &record->blocks[block_id]; ptr = bkpb->bkp_image; - if (bkpb->bimg_info & BKPIMAGE_IS_COMPRESSED) + if (BKPIMAGE_IS_COMPRESSED(bkpb->bimg_info)) { + int compression_method = BKPIMAGE_COMPRESSION(bkpb->bimg_info); /* If a backup block image is compressed, decompress it */ - if (pglz_decompress(ptr, bkpb->bimg_len, tmp.data, - BLCKSZ - bkpb->hole_length, true) < 0) + int32 decomp_result = -1; + switch (compression_method) + { + case WAL_COMPRESSION_PGLZ: + decomp_result = pglz_decompress(ptr, bkpb->bimg_len, tmp.data, + BLCKSZ - bkpb->hole_length, true); + break; + +#ifdef HAVE_LIBZ + case WAL_COMPRESSION_ZLIB: + { + unsigned long decomp_result_l; + decomp_result_l = BLCKSZ - bkpb->hole_length; + if (uncompress((Bytef*)tmp.data, &decomp_result_l, + (Bytef*)ptr, bkpb->bimg_len) == Z_OK) + decomp_result = decomp_result_l; + else + decomp_result = -1; + break; + } +#endif + + default: + report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", + (uint32) (record->ReadRecPtr >> 32), + (uint32) record->ReadRecPtr, + block_id, + compression_method, + wal_compression_name(compression_method)); + return false; + } + + if (decomp_result < 0) { report_invalid_record(record, "invalid compressed image at %X/%X, block %d", LSN_FORMAT_ARGS(record->ReadRecPtr), diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index 68b62d523d..ce1149bed5 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -548,6 +548,7 @@ extern const struct config_enum_entry archive_mode_options[]; extern const struct config_enum_entry recovery_target_action_options[]; extern const struct config_enum_entry sync_method_options[]; extern const struct config_enum_entry dynamic_shared_memory_options[]; +extern const struct config_enum_entry wal_compression_options[]; /* * GUC option variables that are exported from this module @@ -1304,16 +1305,6 @@ static struct config_bool ConfigureNamesBool[] = NULL, NULL, NULL }, - { - {"wal_compression", PGC_SUSET, WAL_SETTINGS, - gettext_noop("Compresses full-page writes written in WAL file."), - NULL - }, - &wal_compression, - false, - NULL, NULL, NULL - }, - { {"wal_init_zero", PGC_SUSET, WAL_SETTINGS, gettext_noop("Writes zeroes to new WAL files before first use."), @@ -4825,6 +4816,16 @@ static struct config_enum ConfigureNamesEnum[] = NULL, NULL, NULL }, + { + {"wal_compression", PGC_SUSET, WAL_SETTINGS, + gettext_noop("Set the method used to compress full page images in the WAL."), + NULL + }, + &wal_compression, + WAL_COMPRESSION_NONE, wal_compression_options, + NULL, NULL, NULL + }, + { {"dynamic_shared_memory_type", PGC_POSTMASTER, RESOURCES_MEM, gettext_noop("Selects the dynamic shared memory implementation used."), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ddbb6dc2be..61786c6e07 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -218,7 +218,7 @@ #full_page_writes = on # recover from partial page writes #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) -#wal_compression = off # enable compression of full-page writes +#wal_compression = off # enable compression of full-page writes: off, pglz, zlib #wal_init_zero = on # zero-fill new WAL files #wal_recycle = on # recycle WAL files #wal_buffers = -1 # min 32kB, -1 sets based on shared_buffers diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index f8b8afe4a7..1cd71ac2f7 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -537,18 +537,21 @@ XLogDumpDisplayRecord(XLogDumpConfig *config, XLogReaderState *record) blk); if (XLogRecHasBlockImage(record, block_id)) { - if (record->blocks[block_id].bimg_info & - BKPIMAGE_IS_COMPRESSED) + if (BKPIMAGE_IS_COMPRESSED(record->blocks[block_id].bimg_info)) { + int compression = BKPIMAGE_COMPRESSION( + record->blocks[block_id].bimg_info); + printf(" (FPW%s); hole: offset: %u, length: %u, " - "compression saved: %u", + "compression method %d/%s, saved: %u", XLogRecBlockImageApply(record, block_id) ? "" : " for WAL verification", record->blocks[block_id].hole_offset, record->blocks[block_id].hole_length, + compression, wal_compression_name(compression), BLCKSZ - - record->blocks[block_id].hole_length - - record->blocks[block_id].bimg_len); + record->blocks[block_id].hole_length - + record->blocks[block_id].bimg_len); } else { diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h index 77187c12be..e8b2c53784 100644 --- a/src/include/access/xlog.h +++ b/src/include/access/xlog.h @@ -116,7 +116,7 @@ extern char *XLogArchiveCommand; extern bool EnableHotStandby; extern bool fullPageWrites; extern bool wal_log_hints; -extern bool wal_compression; +extern int wal_compression; extern bool wal_init_zero; extern bool wal_recycle; extern bool *wal_consistency_checking; diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index 26a743b6b6..8b740af66d 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -324,4 +324,14 @@ extern bool InArchiveRecovery; extern bool StandbyMode; extern char *recoveryRestoreCommand; +/* These are the compression IDs written into bimg_info */ +typedef enum WalCompression +{ + WAL_COMPRESSION_NONE, + WAL_COMPRESSION_PGLZ, + WAL_COMPRESSION_ZLIB, +} WalCompression; + +extern const char *wal_compression_name(WalCompression compression); + #endif /* XLOG_INTERNAL_H */ diff --git a/src/include/access/xlogrecord.h b/src/include/access/xlogrecord.h index 80c92a2498..2a60c0fb92 100644 --- a/src/include/access/xlogrecord.h +++ b/src/include/access/xlogrecord.h @@ -114,7 +114,7 @@ typedef struct XLogRecordBlockHeader * present is (BLCKSZ - ). * * Additionally, when wal_compression is enabled, we will try to compress full - * page images using the PGLZ compression algorithm, after removing the "hole". + * page images, after removing the "hole". * This can reduce the WAL volume, but at some extra cost of CPU spent * on the compression during WAL logging. In this case, since the "hole" * length cannot be calculated by subtracting the number of page image bytes @@ -144,9 +144,18 @@ typedef struct XLogRecordBlockImageHeader /* Information stored in bimg_info */ #define BKPIMAGE_HAS_HOLE 0x01 /* page image has "hole" */ -#define BKPIMAGE_IS_COMPRESSED 0x02 /* page image is compressed */ -#define BKPIMAGE_APPLY 0x04 /* page image should be restored during +#define BKPIMAGE_APPLY 0x02 /* page image should be restored during * replay */ +#define BKPIMAGE_COMPRESS_METHOD1 0x04 /* bits to encode compression method */ +#define BKPIMAGE_COMPRESS_METHOD2 0x08 /* 0=none, 1=pglz, 2=zlib */ + +/* How many bits to shift to extract compression */ +#define BKPIMAGE_COMPRESS_OFFSET_BITS 2 +/* How many bits are for compression */ +#define BKPIMAGE_COMPRESS_BITS 2 +/* Extract the compression from the bimg_info */ +#define BKPIMAGE_COMPRESSION(info) ((info >> BKPIMAGE_COMPRESS_OFFSET_BITS) & ((1<