From 59b269fcde648c02d470c49f74d4edf16cc227b0 Mon Sep 17 00:00:00 2001 From: David Rowley Date: Sat, 6 Apr 2024 21:58:07 +1300 Subject: [PATCH v6 1/2] Enlarge bit-space for MemoryContextMethodID Reserve 4 bits for MemoryContextMethodID rather than 3. 3 bits did technically allow a maximum of 8 memory context types, however, we've opted to reserve some bit patterns which left us with only 4 slots, all of which were used. Here we add another bit which frees up 8 slots for future memory context types. In passing, adjust the enum names in MemoryContextMethodID to make it more clear which ones can be used and which ones are reserved. Author: Matthias van de Meent, David Rowley Discussion: https://postgr.es/m/CAApHDvqGSpCU95TmM=Bp=6xjL_nLys4zdZOpfNyWBk97Xrdj2w@mail.gmail.com --- src/backend/utils/mmgr/README | 21 ++++++++---- src/backend/utils/mmgr/mcxt.c | 41 +++++++++++++----------- src/include/utils/memutils_internal.h | 18 ++++++++--- src/include/utils/memutils_memorychunk.h | 30 +++++++++++++---- 4 files changed, 72 insertions(+), 38 deletions(-) diff --git a/src/backend/utils/mmgr/README b/src/backend/utils/mmgr/README index b20b9d4852..f484f7d6f5 100644 --- a/src/backend/utils/mmgr/README +++ b/src/backend/utils/mmgr/README @@ -395,14 +395,14 @@ relevant MemoryContext as a parameter, operations like free and realloc are trickier. To make those work, we require all memory context types to produce allocated chunks that are immediately, without any padding, preceded by a uint64 value of which the least -significant 3 bits are set to the owning context's MemoryContextMethodID. +significant 4 bits are set to the owning context's MemoryContextMethodID. This allows the code to determine the correct MemoryContextMethods to -use by looking up the mcxt_methods[] array using the 3 bits as an index +use by looking up the mcxt_methods[] array using the 4 bits as an index into that array. If a type of allocator needs additional information about its chunks, like e.g. the size of the allocation, that information can in turn -either be encoded into the remaining 61 bits of the preceding uint64 value +either be encoded into the remaining 60 bits of the preceding uint64 value or if more space is required, additional values may be stored directly prior to the uint64 value. It is up to the context implementation to manage this. @@ -420,13 +420,20 @@ pfree(void *pointer) All of the current memory contexts make use of the MemoryChunk header type which is defined in memutils_memorychunk.h. This suits all of the existing -context types well as it makes use of the remaining 61-bits of the uint64 +context types well as it makes use of the remaining 60-bits of the uint64 header to efficiently encode the size of the chunk of memory (or freelist index, in the case of aset.c) and the number of bytes which must be subtracted from the chunk in order to obtain a reference to the block that the chunk -belongs to. 30 bits are used for each of these. If more than 30 bits are -required then the memory context must manage that itself. This can be done by -calling the MemoryChunkSetHdrMaskExternal() function on the given chunk. +belongs to. 30 bits are used for each of these, but only a total of 59 bits +as the lowest bit for the chunk to block offset is the same bit as the highest +bit of the chunk size. This overlapping is possible as the relative offset +between the block and the chunk is expected to be a MAXALIGNed value which +guarantees the lowest bit is always 0. If more than 30 bits are required for +each of these fields then the memory context must manage that itself. This +can be done by calling the MemoryChunkSetHdrMaskExternal() function on the +given chunk. Whether a chunk is an external chunk can be determined by the 1 +remaining bit from the 64-bit MemoryChunk. + Currently, each memory context type stores large allocations on dedicated blocks (which always contain only a single chunk). For these, finding the block is simple as we know that the chunk must be the first on the given diff --git a/src/backend/utils/mmgr/mcxt.c b/src/backend/utils/mmgr/mcxt.c index 5d426795d9..52ae120af9 100644 --- a/src/backend/utils/mmgr/mcxt.c +++ b/src/backend/utils/mmgr/mcxt.c @@ -37,6 +37,11 @@ static Size BogusGetChunkSpace(void *pointer); /***************************************************************************** * GLOBAL MEMORY * *****************************************************************************/ +#define BOGUS_MCTX(id) \ + [id].free_p = BogusFree, \ + [id].realloc = BogusRealloc, \ + [id].get_chunk_context = BogusGetChunkContext, \ + [id].get_chunk_space = BogusGetChunkSpace static const MemoryContextMethods mcxt_methods[] = { /* aset.c */ @@ -103,27 +108,25 @@ static const MemoryContextMethods mcxt_methods[] = { * invoked from inspection of a chunk (see MCXT_METHOD calls below). */ - [MCTX_UNUSED1_ID].free_p = BogusFree, - [MCTX_UNUSED1_ID].realloc = BogusRealloc, - [MCTX_UNUSED1_ID].get_chunk_context = BogusGetChunkContext, - [MCTX_UNUSED1_ID].get_chunk_space = BogusGetChunkSpace, - - [MCTX_UNUSED2_ID].free_p = BogusFree, - [MCTX_UNUSED2_ID].realloc = BogusRealloc, - [MCTX_UNUSED2_ID].get_chunk_context = BogusGetChunkContext, - [MCTX_UNUSED2_ID].get_chunk_space = BogusGetChunkSpace, - - [MCTX_UNUSED3_ID].free_p = BogusFree, - [MCTX_UNUSED3_ID].realloc = BogusRealloc, - [MCTX_UNUSED3_ID].get_chunk_context = BogusGetChunkContext, - [MCTX_UNUSED3_ID].get_chunk_space = BogusGetChunkSpace, - - [MCTX_UNUSED4_ID].free_p = BogusFree, - [MCTX_UNUSED4_ID].realloc = BogusRealloc, - [MCTX_UNUSED4_ID].get_chunk_context = BogusGetChunkContext, - [MCTX_UNUSED4_ID].get_chunk_space = BogusGetChunkSpace, + BOGUS_MCTX(MCTX_7_UNUSED_ID), + BOGUS_MCTX(MCTX_8_UNUSED_ID), + BOGUS_MCTX(MCTX_9_UNUSED_ID), + BOGUS_MCTX(MCTX_10_UNUSED_ID), + BOGUS_MCTX(MCTX_11_UNUSED_ID), + BOGUS_MCTX(MCTX_12_UNUSED_ID), + BOGUS_MCTX(MCTX_13_UNUSED_ID), + BOGUS_MCTX(MCTX_14_UNUSED_ID), + + /* + * Reserved IDs with bit patterns that we'd see if we were working on + * invalid memory, either uninitialized or wiped. + */ + BOGUS_MCTX(MCTX_0_RESERVED_UNUSEDMEM_ID), + BOGUS_MCTX(MCTX_15_RESERVED_WIPEMEM_ID), }; +#undef BOGUS_MCTX + /* * CurrentMemoryContext * Default memory context for allocations. diff --git a/src/include/utils/memutils_internal.h b/src/include/utils/memutils_internal.h index ad1048fd82..c3f010b595 100644 --- a/src/include/utils/memutils_internal.h +++ b/src/include/utils/memutils_internal.h @@ -104,21 +104,29 @@ extern Size AlignedAllocGetChunkSpace(void *pointer); */ typedef enum MemoryContextMethodID { - MCTX_UNUSED1_ID, /* 000 occurs in never-used memory */ - MCTX_UNUSED2_ID, /* glibc malloc'd chunks usually match 001 */ - MCTX_UNUSED3_ID, /* glibc malloc'd chunks > 128kB match 010 */ + MCTX_0_RESERVED_UNUSEDMEM_ID, /* 0; occurs in never-used memory */ + MCTX_1_RESERVED_GLIBC_ID, /* 1=0001; glibc malloc'd chunks */ + MCTX_2_RESERVED_GLIBC_ID, /* 2=0010; glibc malloc'd chunks > 128kB */ MCTX_ASET_ID, MCTX_GENERATION_ID, MCTX_SLAB_ID, MCTX_ALIGNED_REDIRECT_ID, - MCTX_UNUSED4_ID, /* 111 occurs in wipe_mem'd memory */ + MCTX_7_UNUSED_ID, + MCTX_8_UNUSED_ID, + MCTX_9_UNUSED_ID, + MCTX_10_UNUSED_ID, + MCTX_11_UNUSED_ID, + MCTX_12_UNUSED_ID, + MCTX_13_UNUSED_ID, + MCTX_14_UNUSED_ID, + MCTX_15_RESERVED_WIPEMEM_ID /* 1111 occurs in wipe_mem'd memory (0x7F) */ } MemoryContextMethodID; /* * The number of bits that 8-byte memory chunk headers can use to encode the * MemoryContextMethodID. */ -#define MEMORY_CONTEXT_METHODID_BITS 3 +#define MEMORY_CONTEXT_METHODID_BITS 4 #define MEMORY_CONTEXT_METHODID_MASK \ ((((uint64) 1) << MEMORY_CONTEXT_METHODID_BITS) - 1) diff --git a/src/include/utils/memutils_memorychunk.h b/src/include/utils/memutils_memorychunk.h index 38296abe1b..948a5ac954 100644 --- a/src/include/utils/memutils_memorychunk.h +++ b/src/include/utils/memutils_memorychunk.h @@ -12,7 +12,7 @@ * Although MemoryChunks are used by each of our MemoryContexts, future * implementations may choose to implement their own method for storing chunk * headers. The only requirement is that the header ends with an 8-byte value - * which the least significant 3-bits of are set to the MemoryContextMethodID + * which the least significant 4-bits of are set to the MemoryContextMethodID * of the given context. * * By default, a MemoryChunk is 8 bytes in size, however, when @@ -25,7 +25,7 @@ * used to encode 4 separate pieces of information. Starting with the least * significant bits of 'hdrmask', the bit space is reserved as follows: * - * 1. 3-bits to indicate the MemoryContextMethodID as defined by + * 1. 4-bits to indicate the MemoryContextMethodID as defined by * MEMORY_CONTEXT_METHODID_MASK * 2. 1-bit to denote an "external" chunk (see below) * 3. 30-bits reserved for the MemoryContext to use for anything it @@ -34,6 +34,14 @@ * 4. 30-bits for the number of bytes that must be subtracted from the chunk * to obtain the address of the block that the chunk is stored on. * + * If you're paying close attention, you'll notice this adds up to 65 bits + * rather than 64 bits. This is because the highest order bit of #3 is the + * same bit as the lowest order bit of #4. We can do this as we insist that + * the chunk and block pointers are both MAXALIGNed, therefore the relative + * offset between those will always be a MAXALIGNed value which means the + * lowest order bit is always 0. When fetching the the chunk to block offset + * we mask out the lowest-order bit to ensure it's still zero. + * * In some cases, for example when memory allocations become large, it's * possible fields 3 and 4 above are not large enough to store the values * required for the chunk. In this case, the MemoryContext can choose to mark @@ -93,10 +101,16 @@ */ #define MEMORYCHUNK_MAX_BLOCKOFFSET UINT64CONST(0x3FFFFFFF) +/* + * As above, but mask out the lowest-order (always zero) bit as this is shared + * with the MemoryChunkGetValue field. + */ +#define MEMORYCHUNK_BLOCKOFFSET_MASK UINT64CONST(0x3FFFFFFE) + /* define the least significant base-0 bit of each portion of the hdrmask */ #define MEMORYCHUNK_EXTERNAL_BASEBIT MEMORY_CONTEXT_METHODID_BITS #define MEMORYCHUNK_VALUE_BASEBIT (MEMORYCHUNK_EXTERNAL_BASEBIT + 1) -#define MEMORYCHUNK_BLOCKOFFSET_BASEBIT (MEMORYCHUNK_VALUE_BASEBIT + 30) +#define MEMORYCHUNK_BLOCKOFFSET_BASEBIT (MEMORYCHUNK_VALUE_BASEBIT + 29) /* * A magic number for storing in the free bits of an external chunk. This @@ -131,11 +145,11 @@ typedef struct MemoryChunk (((hdrmask) >> MEMORYCHUNK_VALUE_BASEBIT) & MEMORYCHUNK_MAX_VALUE) /* - * We should have used up all the bits here, so the compiler is likely to - * optimize out the & MEMORYCHUNK_MAX_BLOCKOFFSET. + * We should have used up all the bits here, we just need to ensure we mask + * out the low-order bit that's shared with the MemoryChunkGetValue field. */ #define HdrMaskBlockOffset(hdrmask) \ - (((hdrmask) >> MEMORYCHUNK_BLOCKOFFSET_BASEBIT) & MEMORYCHUNK_MAX_BLOCKOFFSET) + (((hdrmask) >> MEMORYCHUNK_BLOCKOFFSET_BASEBIT) & MEMORYCHUNK_BLOCKOFFSET_MASK) /* For external chunks only, check the magic number matches */ #define HdrMaskCheckMagic(hdrmask) \ @@ -149,6 +163,7 @@ typedef struct MemoryChunk * The number of bytes between 'block' and 'chunk' must be <= * MEMORYCHUNK_MAX_BLOCKOFFSET. * 'value' must be <= MEMORYCHUNK_MAX_VALUE. + * Both 'chunk' and 'block' must be MAXALIGNed pointers. */ static inline void MemoryChunkSetHdrMask(MemoryChunk *chunk, void *block, @@ -157,7 +172,7 @@ MemoryChunkSetHdrMask(MemoryChunk *chunk, void *block, Size blockoffset = (char *) chunk - (char *) block; Assert((char *) chunk >= (char *) block); - Assert(blockoffset <= MEMORYCHUNK_MAX_BLOCKOFFSET); + Assert((blockoffset & MEMORYCHUNK_BLOCKOFFSET_MASK) == blockoffset); Assert(value <= MEMORYCHUNK_MAX_VALUE); Assert((int) methodid <= MEMORY_CONTEXT_METHODID_MASK); @@ -225,6 +240,7 @@ MemoryChunkGetBlock(MemoryChunk *chunk) } /* cleanup all internal definitions */ +#undef MEMORYCHUNK_BLOCKOFFSET_MASK #undef MEMORYCHUNK_EXTERNAL_BASEBIT #undef MEMORYCHUNK_VALUE_BASEBIT #undef MEMORYCHUNK_BLOCKOFFSET_BASEBIT -- 2.40.1.windows.1