From c6809aafd147d0ac286ab73c2d8fbe571c698550 Mon Sep 17 00:00:00 2001 From: Thomas Munro Date: Fri, 26 May 2023 01:41:11 +1200 Subject: [PATCH 1/2] Allow relation segment size to be set by initdb. Previously, relation segment size was a rarely modified compile time option. Make it an initdb option, so that users with very large tables can avoid using so many files and file descriptors. The initdb option --rel-segsize is modeled on the existing --wal-segsize option. The data type used to store the size is int64, not BlockNumber, because it seems reasonable to want to be able to say --rel-segsize=32TB (= don't use segments at all), but that would overflow uint32. The default behavior is unchanged: 1GB segments. On Windows, we can't go above 2GB for now due (we'd have to make a lot of changes due to Windows' small off_t). Discussion: https://postgr.es/m/CA%2BhUKG%2BBGXwMbrvzXAjL8VMGf25y_ga_XnO741g10y0%3Dm6dDiA%40mail.gmail.com diff --git a/configure b/configure index 1b415142d1..a3dee3ea74 100755 --- a/configure +++ b/configure @@ -841,8 +841,6 @@ enable_coverage enable_dtrace enable_tap_tests with_blocksize -with_segsize -with_segsize_blocks with_wal_blocksize with_CC with_llvm @@ -1551,9 +1549,6 @@ Optional Packages: --with-pgport=PORTNUM set default port number [5432] --with-blocksize=BLOCKSIZE set table block size in kB [8] - --with-segsize=SEGSIZE set table segment size in GB [1] - --with-segsize-blocks=SEGSIZE_BLOCKS - set table segment size in blocks [0] --with-wal-blocksize=BLOCKSIZE set WAL block size in kB [8] --with-CC=CMD set compiler (deprecated) @@ -3731,85 +3726,6 @@ cat >>confdefs.h <<_ACEOF _ACEOF -# -# Relation segment size -# - - - -# Check whether --with-segsize was given. -if test "${with_segsize+set}" = set; then : - withval=$with_segsize; - case $withval in - yes) - as_fn_error $? "argument required for --with-segsize option" "$LINENO" 5 - ;; - no) - as_fn_error $? "argument required for --with-segsize option" "$LINENO" 5 - ;; - *) - segsize=$withval - ;; - esac - -else - segsize=1 -fi - - - - - -# Check whether --with-segsize-blocks was given. -if test "${with_segsize_blocks+set}" = set; then : - withval=$with_segsize_blocks; - case $withval in - yes) - as_fn_error $? "argument required for --with-segsize-blocks option" "$LINENO" 5 - ;; - no) - as_fn_error $? "argument required for --with-segsize-blocks option" "$LINENO" 5 - ;; - *) - segsize_blocks=$withval - ;; - esac - -else - segsize_blocks=0 -fi - - - -# If --with-segsize-blocks is non-zero, it is used, --with-segsize -# otherwise. segsize-blocks is only really useful for developers wanting to -# test segment related code. Warn if both are used. -if test $segsize_blocks -ne 0 -a $segsize -ne 1; then - { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: both --with-segsize and --with-segsize-blocks specified, --with-segsize-blocks wins" >&5 -$as_echo "$as_me: WARNING: both --with-segsize and --with-segsize-blocks specified, --with-segsize-blocks wins" >&2;} -fi - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for segment size" >&5 -$as_echo_n "checking for segment size... " >&6; } -if test $segsize_blocks -eq 0; then - # this expression is set up to avoid unnecessary integer overflow - # blocksize is already guaranteed to be a factor of 1024 - RELSEG_SIZE=`expr '(' 1024 / ${blocksize} ')' '*' ${segsize} '*' 1024` - test $? -eq 0 || exit 1 - { $as_echo "$as_me:${as_lineno-$LINENO}: result: ${segsize}GB" >&5 -$as_echo "${segsize}GB" >&6; } -else - RELSEG_SIZE=$segsize_blocks - { $as_echo "$as_me:${as_lineno-$LINENO}: result: ${RELSEG_SIZE} blocks" >&5 -$as_echo "${RELSEG_SIZE} blocks" >&6; } -fi - - -cat >>confdefs.h <<_ACEOF -#define RELSEG_SIZE ${RELSEG_SIZE} -_ACEOF - - # # WAL block size # @@ -15548,13 +15464,6 @@ _ACEOF -# If we don't have largefile support, can't handle segment size >= 2GB. -if test "$ac_cv_sizeof_off_t" -lt 8; then - if expr $RELSEG_SIZE '*' $blocksize '>=' 2 '*' 1024 '*' 1024; then - as_fn_error $? "Large file support is not enabled. Segment size cannot be larger than 1GB." "$LINENO" 5 - fi -fi - # The cast to long int works around a bug in the HP C Compiler # version HP92453-01 B.11.11.23709.GP, which incorrectly rejects # declarations like `int a3[[(sizeof (unsigned char)) >= 0]];'. diff --git a/configure.ac b/configure.ac index 09558ada0f..1c3c7cad4f 100644 --- a/configure.ac +++ b/configure.ac @@ -282,54 +282,6 @@ AC_DEFINE_UNQUOTED([BLCKSZ], ${BLCKSZ}, [ Changing BLCKSZ requires an initdb. ]) -# -# Relation segment size -# -PGAC_ARG_REQ(with, segsize, [SEGSIZE], [set table segment size in GB [1]], - [segsize=$withval], - [segsize=1]) -PGAC_ARG_REQ(with, segsize-blocks, [SEGSIZE_BLOCKS], [set table segment size in blocks [0]], - [segsize_blocks=$withval], - [segsize_blocks=0]) - -# If --with-segsize-blocks is non-zero, it is used, --with-segsize -# otherwise. segsize-blocks is only really useful for developers wanting to -# test segment related code. Warn if both are used. -if test $segsize_blocks -ne 0 -a $segsize -ne 1; then - AC_MSG_WARN([both --with-segsize and --with-segsize-blocks specified, --with-segsize-blocks wins]) -fi - -AC_MSG_CHECKING([for segment size]) -if test $segsize_blocks -eq 0; then - # this expression is set up to avoid unnecessary integer overflow - # blocksize is already guaranteed to be a factor of 1024 - RELSEG_SIZE=`expr '(' 1024 / ${blocksize} ')' '*' ${segsize} '*' 1024` - test $? -eq 0 || exit 1 - AC_MSG_RESULT([${segsize}GB]) -else - RELSEG_SIZE=$segsize_blocks - AC_MSG_RESULT([${RELSEG_SIZE} blocks]) -fi - -AC_DEFINE_UNQUOTED([RELSEG_SIZE], ${RELSEG_SIZE}, [ - RELSEG_SIZE is the maximum number of blocks allowed in one disk file. - Thus, the maximum size of a single file is RELSEG_SIZE * BLCKSZ; - relations bigger than that are divided into multiple files. - - RELSEG_SIZE * BLCKSZ must be less than your OS' limit on file size. - This is often 2 GB or 4GB in a 32-bit operating system, unless you - have large file support enabled. By default, we make the limit 1 GB - to avoid any possible integer-overflow problems within the OS. - A limit smaller than necessary only means we divide a large - relation into more chunks than necessary, so it seems best to err - in the direction of a small limit. - - A power-of-2 value is recommended to save a few cycles in md.c, - but is not absolutely required. - - Changing RELSEG_SIZE requires an initdb. -]) - # # WAL block size # @@ -1757,13 +1709,6 @@ fi dnl Check for largefile support (must be after AC_SYS_LARGEFILE) AC_CHECK_SIZEOF([off_t]) -# If we don't have largefile support, can't handle segment size >= 2GB. -if test "$ac_cv_sizeof_off_t" -lt 8; then - if expr $RELSEG_SIZE '*' $blocksize '>=' 2 '*' 1024 '*' 1024; then - AC_MSG_ERROR([Large file support is not enabled. Segment size cannot be larger than 1GB.]) - fi -fi - AC_CHECK_SIZEOF([bool], [], [#ifdef HAVE_STDBOOL_H #include diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 5da74b3c40..d739577982 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -10877,10 +10877,9 @@ dynamic_library_path = 'C:\tools\postgresql;H:\my_project\lib;$libdir' Reports the number of blocks (pages) that can be stored within a file - segment. It is determined by the value of RELSEG_SIZE - when building the server. The maximum size of a segment file in bytes - is equal to segment_size multiplied by - block_size; by default this is 1GB. + segment. It is changeable with the --rel-segsize option + with a cluster is initialized with initdb. + By default this is 1GB. diff --git a/doc/src/sgml/ref/initdb.sgml b/doc/src/sgml/ref/initdb.sgml index 87945b4b62..18c4bfdaf8 100644 --- a/doc/src/sgml/ref/initdb.sgml +++ b/doc/src/sgml/ref/initdb.sgml @@ -457,6 +457,30 @@ PostgreSQL documentation + + + + + + Set the maximum size of relation segment files. The size must have a suffix + kB, MB, GB or + TB. The default size is 1GB, which was chosen to + support large relations on operating systems without large file support. + This option can only be set during initialization, and cannot be + changed later. + + + + Setting this to a value higher than the default reduces the + number of file descriptors that must be managed while accessing very large + tables. Note that values higher than the file system can support may + result in errors while trying to extend a table (for example Linux ext4 + limits files to 16TB), and values above 2GB are not supported on + operating systems without a large off_t data type + (notably Windows). + + + diff --git a/meson.build b/meson.build index 16b2e86646..e8c6e16e7a 100644 --- a/meson.build +++ b/meson.build @@ -430,16 +430,6 @@ cdata.set('USE_ASSERT_CHECKING', get_option('cassert') ? 1 : false) blocksize = get_option('blocksize').to_int() * 1024 -if get_option('segsize_blocks') != 0 - if get_option('segsize') != 1 - warning('both segsize and segsize_blocks specified, segsize_blocks wins') - endif - - segsize = get_option('segsize_blocks') -else - segsize = (get_option('segsize') * 1024 * 1024 * 1024) / blocksize -endif - cdata.set('BLCKSZ', blocksize, description: '''Size of a disk block --- this also limits the size of a tuple. You can set it bigger if you need bigger tuples (although TOAST should reduce the need @@ -450,7 +440,6 @@ cdata.set('BLCKSZ', blocksize, description: Changing BLCKSZ requires an initdb.''') cdata.set('XLOG_BLCKSZ', get_option('wal_blocksize').to_int() * 1024) -cdata.set('RELSEG_SIZE', segsize) cdata.set('DEF_PGPORT', get_option('pgport')) cdata.set_quoted('DEF_PGPORT_STR', get_option('pgport').to_string()) cdata.set_quoted('PG_KRB_SRVNAM', get_option('krb_srvnam')) @@ -3302,9 +3291,6 @@ if meson.version().version_compare('>=0.57') { 'data block size': '@0@ kB'.format(cdata.get('BLCKSZ') / 1024), 'WAL block size': '@0@ kB'.format(cdata.get('XLOG_BLCKSZ') / 1024), - 'segment size': get_option('segsize_blocks') != 0 ? - '@0@ blocks'.format(cdata.get('RELSEG_SIZE')) : - '@0@ GB'.format(get_option('segsize')), }, section: 'Data layout', ) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index b2430f617c..f441a9051d 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -3901,7 +3901,7 @@ WriteControlFile(void) ControlFile->floatFormat = FLOATFORMAT_VALUE; ControlFile->blcksz = BLCKSZ; - ControlFile->relseg_size = RELSEG_SIZE; + ControlFile->relseg_size = rel_segment_size; ControlFile->xlog_blcksz = XLOG_BLCKSZ; ControlFile->xlog_seg_size = wal_segment_size; @@ -4071,13 +4071,6 @@ ReadControlFile(void) " but the server was compiled with BLCKSZ %d.", ControlFile->blcksz, BLCKSZ), errhint("It looks like you need to recompile or initdb."))); - if (ControlFile->relseg_size != RELSEG_SIZE) - ereport(FATAL, - (errmsg("database files are incompatible with server"), - errdetail("The database cluster was initialized with RELSEG_SIZE %d," - " but the server was compiled with RELSEG_SIZE %d.", - ControlFile->relseg_size, RELSEG_SIZE), - errhint("It looks like you need to recompile or initdb."))); if (ControlFile->xlog_blcksz != XLOG_BLCKSZ) ereport(FATAL, (errmsg("database files are incompatible with server"), @@ -4158,6 +4151,8 @@ ReadControlFile(void) CalculateCheckpointSegments(); + rel_segment_size = ControlFile->relseg_size; + /* Make the initdb settings visible as GUC variables, too */ SetConfigOption("data_checksums", DataChecksumsEnabled() ? "yes" : "no", PGC_INTERNAL, PGC_S_DYNAMIC_DEFAULT); diff --git a/src/backend/backup/basebackup.c b/src/backend/backup/basebackup.c index 45be21131c..d684ce192d 100644 --- a/src/backend/backup/basebackup.c +++ b/src/backend/backup/basebackup.c @@ -40,6 +40,7 @@ #include "storage/fd.h" #include "storage/ipc.h" #include "storage/reinit.h" +#include "storage/smgr.h" #include "utils/builtins.h" #include "utils/guc.h" #include "utils/ps_status.h" @@ -1594,7 +1595,7 @@ sendFile(bbsink *sink, const char *readfilename, const char *tarfilename, */ if (!PageIsNew(page) && PageGetLSN(page) < sink->bbs_state->startptr) { - checksum = pg_checksum_page((char *) page, blkno + segmentno * RELSEG_SIZE); + checksum = pg_checksum_page((char *) page, blkno + segmentno * rel_segment_size); phdr = (PageHeader) page; if (phdr->pd_checksum != checksum) { diff --git a/src/backend/bootstrap/bootstrap.c b/src/backend/bootstrap/bootstrap.c index 49e956b2c5..a90c4281c5 100644 --- a/src/backend/bootstrap/bootstrap.c +++ b/src/backend/bootstrap/bootstrap.c @@ -221,7 +221,7 @@ BootstrapModeMain(int argc, char *argv[], bool check_only) argv++; argc--; - while ((flag = getopt(argc, argv, "B:c:d:D:Fkr:X:-:")) != -1) + while ((flag = getopt(argc, argv, "B:c:d:D:Fkr:X:R:-:")) != -1) { switch (flag) { @@ -279,6 +279,9 @@ BootstrapModeMain(int argc, char *argv[], bool check_only) case 'r': strlcpy(OutputFileName, optarg, MAXPGPATH); break; + case 'R': + rel_segment_size = strtoi64(optarg, NULL, 0); + break; case 'X': { int WalSegSz = strtoul(optarg, NULL, 0); diff --git a/src/backend/storage/file/buffile.c b/src/backend/storage/file/buffile.c index 41ab64100e..7eaf0dc481 100644 --- a/src/backend/storage/file/buffile.c +++ b/src/backend/storage/file/buffile.c @@ -55,9 +55,9 @@ #include "utils/resowner.h" /* - * We break BufFiles into gigabyte-sized segments, regardless of RELSEG_SIZE. - * The reason is that we'd like large BufFiles to be spread across multiple - * tablespaces when available. + * We break BufFiles into gigabyte-sized segments, regardless of + * rel_segment_size. The reason is that we'd like large BufFiles to be spread + * across multiple tablespaces when available. */ #define MAX_PHYSICAL_FILESIZE 0x40000000 #define BUFFILE_SEG_SIZE (MAX_PHYSICAL_FILESIZE / BLCKSZ) diff --git a/src/backend/storage/smgr/md.c b/src/backend/storage/smgr/md.c index 65bb22541c..47801548d4 100644 --- a/src/backend/storage/smgr/md.c +++ b/src/backend/storage/smgr/md.c @@ -32,6 +32,7 @@ #include "pg_trace.h" #include "pgstat.h" #include "postmaster/bgwriter.h" +#include "port/pg_bitutils.h" #include "storage/bufmgr.h" #include "storage/fd.h" #include "storage/md.h" @@ -45,15 +46,15 @@ * The magnetic disk storage manager keeps track of open file * descriptors in its own descriptor pool. This is done to make it * easier to support relations that are larger than the operating - * system's file size limit (often 2GBytes). In order to do that, - * we break relations up into "segment" files that are each shorter than - * the OS file size limit. The segment size is set by the RELSEG_SIZE - * configuration constant in pg_config.h. + * system's file size limit (historically 2GB, sometimes much larger but still + * smaller than the maximum possible relation size). In order to do that, we + * break relations up into "segment" files of a user-specified size chosen at + * initdb time and accessed as rel_segment_size. * * On disk, a relation must consist of consecutively numbered segment * files in the pattern - * -- Zero or more full segments of exactly RELSEG_SIZE blocks each - * -- Exactly one partial segment of size 0 <= size < RELSEG_SIZE blocks + * -- Zero or more full segments of exactly rel_segment_size blocks each + * -- Exactly one partial segment of size 0 <= size < rel_segment_size blocks * -- Optionally, any number of inactive segments of size 0 blocks. * The full and partial segments are collectively the "active" segments. * Inactive segments are those that once contained data but are currently @@ -110,7 +111,7 @@ static MemoryContext MdCxt; /* context for all MdfdVec objects */ #define EXTENSION_CREATE_RECOVERY (1 << 3) /* * Allow opening segments which are preceded by segments smaller than - * RELSEG_SIZE, e.g. inactive segments (see above). Note that this breaks + * rel_segment_size, e.g. inactive segments (see above). Note that this breaks * mdnblocks() and related functionality henceforth - which currently is ok, * because this is only required in the checkpointer which never uses * mdnblocks(). @@ -142,6 +143,31 @@ static MdfdVec *_mdfd_getseg(SMgrRelation reln, ForkNumber forknum, static BlockNumber _mdnblocks(SMgrRelation reln, ForkNumber forknum, MdfdVec *seg); +/* Given a block number, which segment is it in? */ +static inline uint32 +blockno_to_segno(BlockNumber blockno) +{ + /* Because it's a power of two, we can use a shift instead of "/". */ + Assert(pg_popcount64(rel_segment_size) == 1); + return (uint64) blockno >> pg_leftmost_one_pos64(rel_segment_size); +} + +/* Given a block number, which block is that within its segment? */ +static inline BlockNumber +blockno_within_segment(BlockNumber blockno) +{ + /* Because it's a power of two, we can use a mask instead of "%". */ + Assert(pg_popcount64(rel_segment_size) == 1); + return blockno & (rel_segment_size - 1); +} + +/* Given a block number, convert it to byte offset within a segment. */ +static inline off_t +blockno_to_seekpos(BlockNumber blockno) +{ + return blockno_within_segment(blockno) * (off_t) BLCKSZ; +} + static inline int _mdfd_open_flags(void) { @@ -487,9 +513,9 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, v = _mdfd_getseg(reln, forknum, blocknum, skipFsync, EXTENSION_CREATE); - seekpos = (off_t) BLCKSZ * (blocknum % ((BlockNumber) RELSEG_SIZE)); + seekpos = blockno_to_seekpos(blocknum); - Assert(seekpos < (off_t) BLCKSZ * RELSEG_SIZE); + Assert(seekpos < (off_t) BLCKSZ * rel_segment_size); if ((nbytes = FileWrite(v->mdfd_vfd, buffer, BLCKSZ, seekpos, WAIT_EVENT_DATA_FILE_EXTEND)) != BLCKSZ) { @@ -511,7 +537,7 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, if (!skipFsync && !SmgrIsTemp(reln)) register_dirty_segment(reln, forknum, v); - Assert(_mdnblocks(reln, forknum, v) <= ((BlockNumber) RELSEG_SIZE)); + Assert(_mdnblocks(reln, forknum, v) <= rel_segment_size); } /* @@ -549,19 +575,19 @@ mdzeroextend(SMgrRelation reln, ForkNumber forknum, while (remblocks > 0) { - BlockNumber segstartblock = curblocknum % ((BlockNumber) RELSEG_SIZE); - off_t seekpos = (off_t) BLCKSZ * segstartblock; + BlockNumber segstartblock = blockno_within_segment(blocknum); + off_t seekpos = blockno_to_seekpos(blocknum); int numblocks; - if (segstartblock + remblocks > RELSEG_SIZE) - numblocks = RELSEG_SIZE - segstartblock; + if (segstartblock + remblocks > rel_segment_size) + numblocks = rel_segment_size - segstartblock; else numblocks = remblocks; v = _mdfd_getseg(reln, forknum, curblocknum, skipFsync, EXTENSION_CREATE); - Assert(segstartblock < RELSEG_SIZE); - Assert(segstartblock + numblocks <= RELSEG_SIZE); + Assert(segstartblock < rel_segment_size); + Assert(segstartblock + numblocks <= rel_segment_size); /* * If available and useful, use posix_fallocate() (via FileAllocate()) @@ -615,7 +641,7 @@ mdzeroextend(SMgrRelation reln, ForkNumber forknum, if (!skipFsync && !SmgrIsTemp(reln)) register_dirty_segment(reln, forknum, v); - Assert(_mdnblocks(reln, forknum, v) <= ((BlockNumber) RELSEG_SIZE)); + Assert(_mdnblocks(reln, forknum, v) <= rel_segment_size); remblocks -= numblocks; curblocknum += numblocks; @@ -667,7 +693,7 @@ mdopenfork(SMgrRelation reln, ForkNumber forknum, int behavior) mdfd->mdfd_vfd = fd; mdfd->mdfd_segno = 0; - Assert(_mdnblocks(reln, forknum, mdfd) <= ((BlockNumber) RELSEG_SIZE)); + Assert(_mdnblocks(reln, forknum, mdfd) <= rel_segment_size); return mdfd; } @@ -723,9 +749,9 @@ mdprefetch(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum) if (v == NULL) return false; - seekpos = (off_t) BLCKSZ * (blocknum % ((BlockNumber) RELSEG_SIZE)); + seekpos = blockno_to_seekpos(blocknum); - Assert(seekpos < (off_t) BLCKSZ * RELSEG_SIZE); + Assert(seekpos < (off_t) BLCKSZ * rel_segment_size); (void) FilePrefetch(v->mdfd_vfd, seekpos, BLCKSZ, WAIT_EVENT_DATA_FILE_PREFETCH); #endif /* USE_PREFETCH */ @@ -757,9 +783,9 @@ mdread(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, v = _mdfd_getseg(reln, forknum, blocknum, false, EXTENSION_FAIL | EXTENSION_CREATE_RECOVERY); - seekpos = (off_t) BLCKSZ * (blocknum % ((BlockNumber) RELSEG_SIZE)); + seekpos = blockno_to_seekpos(blocknum); - Assert(seekpos < (off_t) BLCKSZ * RELSEG_SIZE); + Assert(seekpos < (off_t) BLCKSZ * rel_segment_size); nbytes = FileRead(v->mdfd_vfd, buffer, BLCKSZ, seekpos, WAIT_EVENT_DATA_FILE_READ); @@ -831,9 +857,9 @@ mdwrite(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, v = _mdfd_getseg(reln, forknum, blocknum, skipFsync, EXTENSION_FAIL | EXTENSION_CREATE_RECOVERY); - seekpos = (off_t) BLCKSZ * (blocknum % ((BlockNumber) RELSEG_SIZE)); + seekpos = blockno_to_seekpos(blocknum); - Assert(seekpos < (off_t) BLCKSZ * RELSEG_SIZE); + Assert(seekpos < (off_t) BLCKSZ * rel_segment_size); nbytes = FileWrite(v->mdfd_vfd, buffer, BLCKSZ, seekpos, WAIT_EVENT_DATA_FILE_WRITE); @@ -904,17 +930,17 @@ mdwriteback(SMgrRelation reln, ForkNumber forknum, return; /* compute offset inside the current segment */ - segnum_start = blocknum / RELSEG_SIZE; + segnum_start = blockno_to_segno(blocknum); /* compute number of desired writes within the current segment */ - segnum_end = (blocknum + nblocks - 1) / RELSEG_SIZE; + segnum_end = blockno_to_segno(blocknum + nblocks - 1); if (segnum_start != segnum_end) - nflush = RELSEG_SIZE - (blocknum % ((BlockNumber) RELSEG_SIZE)); + nflush = rel_segment_size - blockno_within_segment(blocknum); Assert(nflush >= 1); Assert(nflush <= nblocks); - seekpos = (off_t) BLCKSZ * (blocknum % ((BlockNumber) RELSEG_SIZE)); + seekpos = blockno_to_seekpos(blocknum); FileWriteback(v->mdfd_vfd, seekpos, (off_t) BLCKSZ * nflush, WAIT_EVENT_DATA_FILE_FLUSH); @@ -945,8 +971,8 @@ mdnblocks(SMgrRelation reln, ForkNumber forknum) /* * Start from the last open segments, to avoid redundant seeks. We have - * previously verified that these segments are exactly RELSEG_SIZE long, - * and it's useless to recheck that each time. + * previously verified that these segments are exactly rel_segment_size + * long, and it's useless to recheck that each time. * * NOTE: this assumption could only be wrong if another backend has * truncated the relation. We rely on higher code levels to handle that @@ -962,13 +988,13 @@ mdnblocks(SMgrRelation reln, ForkNumber forknum) for (;;) { nblocks = _mdnblocks(reln, forknum, v); - if (nblocks > ((BlockNumber) RELSEG_SIZE)) + if (nblocks > rel_segment_size) elog(FATAL, "segment too big"); - if (nblocks < ((BlockNumber) RELSEG_SIZE)) - return (segno * ((BlockNumber) RELSEG_SIZE)) + nblocks; + if (nblocks < rel_segment_size) + return (segno * rel_segment_size) + nblocks; /* - * If segment is exactly RELSEG_SIZE, advance to next one. + * If segment is exactly rel_segment_size, advance to next one. */ segno++; @@ -981,7 +1007,7 @@ mdnblocks(SMgrRelation reln, ForkNumber forknum) */ v = _mdfd_openseg(reln, forknum, segno, 0); if (v == NULL) - return segno * ((BlockNumber) RELSEG_SIZE); + return segno * rel_segment_size; } } @@ -1022,7 +1048,7 @@ mdtruncate(SMgrRelation reln, ForkNumber forknum, BlockNumber nblocks) { MdfdVec *v; - priorblocks = (curopensegs - 1) * RELSEG_SIZE; + priorblocks = (curopensegs - 1) * rel_segment_size; v = &reln->md_seg_fds[forknum][curopensegs - 1]; @@ -1047,13 +1073,13 @@ mdtruncate(SMgrRelation reln, ForkNumber forknum, BlockNumber nblocks) FileClose(v->mdfd_vfd); _fdvec_resize(reln, forknum, curopensegs - 1); } - else if (priorblocks + ((BlockNumber) RELSEG_SIZE) > nblocks) + else if (priorblocks + rel_segment_size > nblocks) { /* * This is the last segment we want to keep. Truncate the file to * the right length. NOTE: if nblocks is exactly a multiple K of - * RELSEG_SIZE, we will truncate the K+1st segment to 0 length but - * keep it. This adheres to the invariant given in the header + * rel_setment_size, we will truncate the K+1st segment to 0 length + * but keep it. This adheres to the invariant given in the header * comments. */ BlockNumber lastsegblocks = nblocks - priorblocks; @@ -1369,7 +1395,7 @@ _mdfd_openseg(SMgrRelation reln, ForkNumber forknum, BlockNumber segno, v->mdfd_vfd = fd; v->mdfd_segno = segno; - Assert(_mdnblocks(reln, forknum, v) <= ((BlockNumber) RELSEG_SIZE)); + Assert(_mdnblocks(reln, forknum, v) <= rel_segment_size); /* all done */ return v; @@ -1396,7 +1422,7 @@ _mdfd_getseg(SMgrRelation reln, ForkNumber forknum, BlockNumber blkno, (EXTENSION_FAIL | EXTENSION_CREATE | EXTENSION_RETURN_NULL | EXTENSION_DONT_OPEN)); - targetseg = blkno / ((BlockNumber) RELSEG_SIZE); + targetseg = blockno_to_segno(blkno); /* if an existing and opened segment, we're done */ if (targetseg < reln->md_num_open_segs[forknum]) @@ -1433,7 +1459,7 @@ _mdfd_getseg(SMgrRelation reln, ForkNumber forknum, BlockNumber blkno, Assert(nextsegno == v->mdfd_segno + 1); - if (nblocks > ((BlockNumber) RELSEG_SIZE)) + if (nblocks > rel_segment_size) elog(FATAL, "segment too big"); if ((behavior & EXTENSION_CREATE) || @@ -1448,31 +1474,31 @@ _mdfd_getseg(SMgrRelation reln, ForkNumber forknum, BlockNumber blkno, * ahead and create the segments so we can finish out the replay. * * We have to maintain the invariant that segments before the last - * active segment are of size RELSEG_SIZE; therefore, if + * active segment are of size rel_segment_size; therefore, if * extending, pad them out with zeroes if needed. (This only * matters if in recovery, or if the caller is extending the * relation discontiguously, but that can happen in hash indexes.) */ - if (nblocks < ((BlockNumber) RELSEG_SIZE)) + if (nblocks < rel_segment_size) { char *zerobuf = palloc_aligned(BLCKSZ, PG_IO_ALIGN_SIZE, MCXT_ALLOC_ZERO); mdextend(reln, forknum, - nextsegno * ((BlockNumber) RELSEG_SIZE) - 1, + nextsegno * rel_segment_size - 1, zerobuf, skipFsync); pfree(zerobuf); } flags = O_CREAT; } else if (!(behavior & EXTENSION_DONT_CHECK_SIZE) && - nblocks < ((BlockNumber) RELSEG_SIZE)) + nblocks < rel_segment_size) { /* * When not extending (or explicitly including truncated * segments), only open the next segment if the current one is - * exactly RELSEG_SIZE. If not (this branch), either return NULL - * or fail. + * exactly rel_segment_size. If not (this branch), either return + * NULL or fail. */ if (behavior & EXTENSION_RETURN_NULL) { diff --git a/src/backend/storage/smgr/smgr.c b/src/backend/storage/smgr/smgr.c index f76c4605db..5cada9f130 100644 --- a/src/backend/storage/smgr/smgr.c +++ b/src/backend/storage/smgr/smgr.c @@ -24,10 +24,18 @@ #include "storage/ipc.h" #include "storage/md.h" #include "storage/smgr.h" +#include "utils/guc_tables.h" #include "utils/hsearch.h" #include "utils/inval.h" +/* + * The number of blocks that should be in a segment file. Has a wider type + * than BlockNumber, so that can represent the case the whole relation fits in + * one file. + */ +int64 rel_segment_size; + /* * This struct of function pointers defines the API between smgr.c and * any individual storage manager module. Note that smgr subfunctions are @@ -764,3 +772,9 @@ ProcessBarrierSmgrRelease(void) smgrreleaseall(); return true; } + +const char * +show_segment_size(void) +{ + return ShowGUCInt64WithUnits(rel_segment_size, GUC_UNIT_BLOCKS); +} diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index a9033b7a54..c9d6f732f8 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -5273,6 +5273,22 @@ GetConfigOptionByName(const char *name, const char **varname, bool missing_ok) return ShowGUCOption(record, true); } +/* + * Show unit-based values with appropriate unit, as ShowGUCOption() would. + * This can be used by custom show hooks. + */ +char * +ShowGUCInt64WithUnits(int64 value, int flags) +{ + int64 number; + const char *unit; + char buffer[256]; + + convert_int_from_base_unit(value, flags & GUC_UNIT, &number, &unit); + snprintf(buffer, sizeof(buffer), INT64_FORMAT "%s", number, unit); + return pstrdup(buffer); +} + /* * ShowGUCOption: get string value of variable * diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 68aecad66f..3794b9dc15 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -586,10 +586,10 @@ static int max_function_args; static int max_index_keys; static int max_identifier_length; static int block_size; -static int segment_size; static int shared_memory_size_mb; static int shared_memory_size_in_huge_pages; static int wal_block_size; +static int phony_segment_size; static bool data_checksums; static bool integer_datetimes; @@ -3125,15 +3125,19 @@ struct config_int ConfigureNamesInt[] = NULL, NULL, NULL }, + /* + * We used a phony GUC with a custome show function, because we don't + * support GUCs with a wide enough type. + */ { {"segment_size", PGC_INTERNAL, PRESET_OPTIONS, gettext_noop("Shows the number of pages per disk file."), NULL, GUC_UNIT_BLOCKS | GUC_NOT_IN_SAMPLE | GUC_DISALLOW_IN_FILE }, - &segment_size, - RELSEG_SIZE, RELSEG_SIZE, RELSEG_SIZE, - NULL, NULL, NULL + &phony_segment_size, + 0, 0, 0, + NULL, NULL, show_segment_size }, { diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c index 09a5c98cc0..ecb6950c35 100644 --- a/src/bin/initdb/initdb.c +++ b/src/bin/initdb/initdb.c @@ -80,6 +80,7 @@ #include "getopt_long.h" #include "mb/pg_wchar.h" #include "miscadmin.h" +#include "port/pg_bitutils.h" /* Ideally this would be in a .h file, but it hardly seems worth the trouble */ @@ -169,6 +170,8 @@ static bool data_checksums = false; static char *xlog_dir = NULL; static char *str_wal_segment_size_mb = NULL; static int wal_segment_size_mb; +static char *str_rel_segment_size = NULL; +static int64 rel_segment_size; /* internal vars */ @@ -1535,9 +1538,10 @@ bootstrap_template1(void) unsetenv("PGCLIENTENCODING"); snprintf(cmd, sizeof(cmd), - "\"%s\" --boot -X %d %s %s %s %s", + "\"%s\" --boot -X %d -R " INT64_FORMAT " %s %s %s %s", backend_exec, wal_segment_size_mb * (1024 * 1024), + rel_segment_size, data_checksums ? "-k" : "", boot_options, extra_options, debug ? "-d 5" : ""); @@ -2481,6 +2485,7 @@ usage(const char *progname) printf(_(" -W, --pwprompt prompt for a password for the new superuser\n")); printf(_(" -X, --waldir=WALDIR location for the write-ahead log directory\n")); printf(_(" --wal-segsize=SIZE size of WAL segments, in megabytes\n")); + printf(_(" --rel-segsize=SIZE size of relation segments\n")); printf(_("\nLess commonly used options:\n")); printf(_(" -c, --set NAME=VALUE override default setting for server parameter\n")); printf(_(" -d, --debug generate lots of debugging output\n")); @@ -3129,6 +3134,7 @@ main(int argc, char *argv[]) {"locale-provider", required_argument, NULL, 15}, {"icu-locale", required_argument, NULL, 16}, {"icu-rules", required_argument, NULL, 17}, + {"rel-segsize", required_argument, NULL, 18}, {NULL, 0, NULL, 0} }; @@ -3309,6 +3315,9 @@ main(int argc, char *argv[]) case 17: icu_rules = pg_strdup(optarg); break; + case 18: + str_rel_segment_size = pg_strdup(optarg); + break; default: /* getopt_long already emitted a complaint */ pg_log_error_hint("Try \"%s --help\" for more information.", progname); @@ -3389,6 +3398,43 @@ main(int argc, char *argv[]) pg_fatal("argument of --wal-segsize must be a power of 2 between 1 and 1024"); } + /* set rel segment size */ + if (str_rel_segment_size == NULL) + { + rel_segment_size = (1024 * 1024 * 1024) / BLCKSZ; + } + else + { + int64 bytes; + char *endptr; + + bytes = strtol(str_rel_segment_size, &endptr, 10); + if (endptr == str_rel_segment_size) + pg_fatal("argument of --rel-segsize must begin with a number"); + if (bytes == 0) + pg_fatal("argument of --rel-segsize must be greater than zero"); + + if (strcmp(endptr, "kB") == 0) + bytes *= 1024; + else if (strcmp(endptr, "MB") == 0) + bytes *= 1024 * 1024; + else if (strcmp(endptr, "GB") == 0) + bytes *= 1024 * 1024 * 1024; + else if (strcmp(endptr, "TB") == 0) + bytes *= UINT64CONST(1024) * 1024 * 1024 * 1024; + else + pg_fatal("argument of --rel-segsize must end with kB, MB, GB or TB"); + + if (bytes % BLCKSZ != 0) + pg_fatal("argument of --rel-segsize must be a multiple of BLCKSZ"); + if (pg_popcount64(bytes) != 1) + pg_fatal("argument of --rel-segsize must be a power of two"); + if (sizeof(off_t) < 8 && bytes > (1 << 31)) + pg_fatal("argument of --rel-segsize is too large for this platform's off_t"); + + rel_segment_size = bytes / BLCKSZ; + } + get_restricted_token(); setup_pgdata(); diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c index 19eb67e485..8685f03bf2 100644 --- a/src/bin/pg_checksums/pg_checksums.c +++ b/src/bin/pg_checksums/pg_checksums.c @@ -231,7 +231,7 @@ scan_file(const char *fn, int segmentno) if (PageIsNew(buf.data)) continue; - csum = pg_checksum_page(buf.data, blockno + segmentno * RELSEG_SIZE); + csum = pg_checksum_page(buf.data, blockno + segmentno * ControlFile->relseg_size); if (mode == PG_MODE_CHECK) { if (csum != header->pd_checksum) diff --git a/src/bin/pg_controldata/pg_controldata.c b/src/bin/pg_controldata/pg_controldata.c index c390ec51ce..ff5aaf43ff 100644 --- a/src/bin/pg_controldata/pg_controldata.c +++ b/src/bin/pg_controldata/pg_controldata.c @@ -305,7 +305,7 @@ main(int argc, char *argv[]) /* we don't print floatFormat since can't say much useful about it */ printf(_("Database block size: %u\n"), ControlFile->blcksz); - printf(_("Blocks per segment of large relation: %u\n"), + printf(_("Blocks per segment of large relation: " INT64_FORMAT "\n"), ControlFile->relseg_size); printf(_("WAL block size: %u\n"), ControlFile->xlog_blcksz); diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c index e7ef2b8bd0..2dcd886371 100644 --- a/src/bin/pg_resetwal/pg_resetwal.c +++ b/src/bin/pg_resetwal/pg_resetwal.c @@ -683,7 +683,7 @@ GuessControlValues(void) ControlFile.maxAlign = MAXIMUM_ALIGNOF; ControlFile.floatFormat = FLOATFORMAT_VALUE; ControlFile.blcksz = BLCKSZ; - ControlFile.relseg_size = RELSEG_SIZE; + ControlFile.relseg_size = 1024 * 1024 * 1024; ControlFile.xlog_blcksz = XLOG_BLCKSZ; ControlFile.xlog_seg_size = DEFAULT_XLOG_SEG_SIZE; ControlFile.nameDataLen = NAMEDATALEN; @@ -751,7 +751,7 @@ PrintControlValues(bool guessed) /* we don't print floatFormat since can't say much useful about it */ printf(_("Database block size: %u\n"), ControlFile.blcksz); - printf(_("Blocks per segment of large relation: %u\n"), + printf(_("Blocks per segment of large relation: " INT64_FORMAT "\n"), ControlFile.relseg_size); printf(_("WAL block size: %u\n"), ControlFile.xlog_blcksz); diff --git a/src/bin/pg_rewind/filemap.c b/src/bin/pg_rewind/filemap.c index bd5c598e20..693ee195ed 100644 --- a/src/bin/pg_rewind/filemap.c +++ b/src/bin/pg_rewind/filemap.c @@ -296,8 +296,8 @@ process_target_wal_block_change(ForkNumber forknum, RelFileLocator rlocator, BlockNumber blkno_inseg; int segno; - segno = blkno / RELSEG_SIZE; - blkno_inseg = blkno % RELSEG_SIZE; + segno = blkno / rel_segment_size; + blkno_inseg = blkno % rel_segment_size;; path = datasegpath(rlocator, forknum, segno); entry = lookup_filehash_entry(path); diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index f7f3b8227f..f3db47ca04 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -61,6 +61,7 @@ static ControlFileData ControlFile_source_after; const char *progname; int WalSegSz; +int64 rel_segment_size; /* Configuration options */ char *datadir_target = NULL; @@ -1028,6 +1029,8 @@ digestControlFile(ControlFileData *ControlFile, const char *content, WalSegSz), WalSegSz); + rel_segment_size = ControlFile->relseg_size; + /* Additional checks on control file */ checkControlFile(ControlFile); } diff --git a/src/bin/pg_rewind/pg_rewind.h b/src/bin/pg_rewind/pg_rewind.h index ef8bdc1fbb..04e84f393b 100644 --- a/src/bin/pg_rewind/pg_rewind.h +++ b/src/bin/pg_rewind/pg_rewind.h @@ -24,6 +24,7 @@ extern bool showprogress; extern bool dry_run; extern bool do_sync; extern int WalSegSz; +extern int64 rel_segment_size; /* Target history */ extern TimeLineHistoryEntry *targetHistory; diff --git a/src/bin/pg_upgrade/relfilenumber.c b/src/bin/pg_upgrade/relfilenumber.c index 34bc9c1504..f0760ed522 100644 --- a/src/bin/pg_upgrade/relfilenumber.c +++ b/src/bin/pg_upgrade/relfilenumber.c @@ -180,7 +180,7 @@ transfer_relfile(FileNameMap *map, const char *type_suffix, bool vm_must_add_fro /* * Now copy/link any related segments as well. Remember, PG breaks large - * files into 1GB segments, the first segment has no extension, subsequent + * files into segments, the first segment has no extension, subsequent * segments are named relfilenumber.1, relfilenumber.2, relfilenumber.3. */ for (segno = 0;; segno++) diff --git a/src/include/catalog/pg_control.h b/src/include/catalog/pg_control.h index dc953977c5..6a66494c2e 100644 --- a/src/include/catalog/pg_control.h +++ b/src/include/catalog/pg_control.h @@ -203,7 +203,7 @@ typedef struct ControlFileData * compatible with the backend executable. */ uint32 blcksz; /* data block size for this DB */ - uint32 relseg_size; /* blocks per segment of large relation */ + int64 relseg_size; /* blocks per segment of large relation */ uint32 xlog_blcksz; /* block size within WAL files */ uint32 xlog_seg_size; /* size of each WAL segment */ diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 6d572c3820..8ec9cc9b9f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -659,19 +659,6 @@ your system. */ #undef PTHREAD_CREATE_JOINABLE -/* RELSEG_SIZE is the maximum number of blocks allowed in one disk file. Thus, - the maximum size of a single file is RELSEG_SIZE * BLCKSZ; relations bigger - than that are divided into multiple files. RELSEG_SIZE * BLCKSZ must be - less than your OS' limit on file size. This is often 2 GB or 4GB in a - 32-bit operating system, unless you have large file support enabled. By - default, we make the limit 1 GB to avoid any possible integer-overflow - problems within the OS. A limit smaller than necessary only means we divide - a large relation into more chunks than necessary, so it seems best to err - in the direction of a small limit. A power-of-2 value is recommended to - save a few cycles in md.c, but is not absolutely required. Changing - RELSEG_SIZE requires an initdb. */ -#undef RELSEG_SIZE - /* The size of `bool', as computed by sizeof. */ #undef SIZEOF_BOOL diff --git a/src/include/storage/smgr.h b/src/include/storage/smgr.h index a9a179aaba..7a02a13e14 100644 --- a/src/include/storage/smgr.h +++ b/src/include/storage/smgr.h @@ -18,6 +18,8 @@ #include "storage/block.h" #include "storage/relfilelocator.h" +extern int64 rel_segment_size; + /* * smgr.c maintains a table of SMgrRelation objects, which are essentially * cached file handles. An SMgrRelation is created (if not already present) @@ -109,5 +111,6 @@ extern void smgrtruncate(SMgrRelation reln, ForkNumber *forknum, extern void smgrimmedsync(SMgrRelation reln, ForkNumber forknum); extern void AtEOXact_SMgr(void); extern bool ProcessBarrierSmgrRelease(void); +extern const char *show_segment_size(void); #endif /* SMGR_H */ diff --git a/src/include/utils/guc_tables.h b/src/include/utils/guc_tables.h index d5a0880678..9514f6c1a5 100644 --- a/src/include/utils/guc_tables.h +++ b/src/include/utils/guc_tables.h @@ -291,6 +291,7 @@ extern struct config_generic **get_explain_guc_options(int *num); /* get string value of variable */ extern char *ShowGUCOption(struct config_generic *record, bool use_units); +extern char *ShowGUCInt64WithUnits(int64 value, int flags); /* get whether or not the GUC variable is visible to current user */ extern bool ConfigOptionIsVisible(struct config_generic *conf); diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index b6d31c3583..909de3bb9a 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -415,8 +415,6 @@ sub GenerateFiles qq{"PostgreSQL $package_version$extraver, compiled by Visual C++ build " CppAsString2(_MSC_VER) ", $bits-bit"}, PROFILE_PID_DIR => undef, PTHREAD_CREATE_JOINABLE => undef, - RELSEG_SIZE => (1024 / $self->{options}->{blocksize}) * - $self->{options}->{segsize} * 1024, SIZEOF_BOOL => 1, SIZEOF_LONG => 4, SIZEOF_OFF_T => undef, -- 2.39.2