diff --git a/src/backend/access/common/heaptuple.c b/src/backend/access/common/heaptuple.c index e27ec78..631e555 100644 --- a/src/backend/access/common/heaptuple.c +++ b/src/backend/access/common/heaptuple.c @@ -741,7 +741,9 @@ heap_form_tuple(TupleDesc tupleDescriptor, * Allocate and zero the space needed. Note that the tuple body and * HeapTupleData management structure are allocated in one chunk. */ - tuple = (HeapTuple) palloc0(HEAPTUPLESIZE + len); + tuple = MemoryContextAllocExtended(CurrentMemoryContext, + HEAPTUPLESIZE + len, + MCXT_ALLOC_HUGE | MCXT_ALLOC_ZERO); tuple->t_data = td = (HeapTupleHeader) ((char *) tuple + HEAPTUPLESIZE); /* diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index 3c81906..ec5d6f1 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -385,7 +385,7 @@ ReceiveCopyBegin(CopyState cstate) pq_sendint(&buf, format, 2); /* per-column formats */ pq_endmessage(&buf); cstate->copy_dest = COPY_NEW_FE; - cstate->fe_msgbuf = makeStringInfo(); + cstate->fe_msgbuf = makeLongStringInfo(); } else { @@ -1907,7 +1907,7 @@ CopyTo(CopyState cstate) cstate->null_print_client = cstate->null_print; /* default */ /* We use fe_msgbuf as a per-row buffer regardless of copy_dest */ - cstate->fe_msgbuf = makeStringInfo(); + cstate->fe_msgbuf = makeLongStringInfo(); /* Get info about the columns we need to process. */ cstate->out_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo)); @@ -2742,8 +2742,8 @@ BeginCopyFrom(ParseState *pstate, cstate->cur_attval = NULL; /* Set up variables to avoid per-attribute overhead. */ - initStringInfo(&cstate->attribute_buf); - initStringInfo(&cstate->line_buf); + initLongStringInfo(&cstate->attribute_buf); + initLongStringInfo(&cstate->line_buf); cstate->line_buf_converted = false; cstate->raw_buf = (char *) palloc(RAW_BUF_SIZE + 1); cstate->raw_buf_index = cstate->raw_buf_len = 0; diff --git a/src/backend/lib/stringinfo.c b/src/backend/lib/stringinfo.c index 7382e08..35f2eab 100644 --- a/src/backend/lib/stringinfo.c +++ b/src/backend/lib/stringinfo.c @@ -37,10 +37,28 @@ makeStringInfo(void) } /* + * makeLongStringInfo + * + * Same as makeStringInfo for larger strings. + */ +StringInfo +makeLongStringInfo(void) +{ + StringInfo res; + + res = (StringInfo) palloc(sizeof(StringInfoData)); + + initLongStringInfo(res); + + return res; +} + + +/* * initStringInfo * * Initialize a StringInfoData struct (with previously undefined contents) - * to describe an empty string. + * to describe an empty string; don't enable long strings yet. */ void initStringInfo(StringInfo str) @@ -49,10 +67,23 @@ initStringInfo(StringInfo str) str->data = (char *) palloc(size); str->maxlen = size; + str->long_ok = false; resetStringInfo(str); } /* + * initLongStringInfo + * + * As initStringInfo, plus enable long strings. + */ +void +initLongStringInfo(StringInfo str) +{ + initStringInfo(str); + str->long_ok = true; +} + +/* * resetStringInfo * * Reset the StringInfo: the data buffer remains valid, but its @@ -142,7 +173,7 @@ appendStringInfoVA(StringInfo str, const char *fmt, va_list args) /* * Return pvsnprintf's estimate of the space needed. (Although this is * given as a size_t, we know it will fit in int because it's not more - * than MaxAllocSize.) + * than either MaxAllocSize or half an int's width.) */ return (int) nprinted; } @@ -244,7 +275,16 @@ appendBinaryStringInfo(StringInfo str, const char *data, int datalen) void enlargeStringInfo(StringInfo str, int needed) { - int newlen; + Size newlen; + Size limit; + + /* + * Determine the upper size limit. The fact that the StringInfo API uses + * "int" everywhere limits us to int's width even for "long_ok" strings. + */ + limit = str->long_ok ? + (((Size) 1) << (Min(sizeof(int), sizeof(Size)) * 8 - 1)) - 1 : + MaxAllocSize; /* * Guard against out-of-range "needed" values. Without this, we can get @@ -252,7 +292,7 @@ enlargeStringInfo(StringInfo str, int needed) */ if (needed < 0) /* should not happen */ elog(ERROR, "invalid string enlargement request size: %d", needed); - if (((Size) needed) >= (MaxAllocSize - (Size) str->len)) + if (((Size) needed) >= (limit - (Size) str->len)) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("out of memory"), @@ -261,7 +301,7 @@ enlargeStringInfo(StringInfo str, int needed) needed += str->len + 1; /* total space required now */ - /* Because of the above test, we now have needed <= MaxAllocSize */ + /* Because of the above test, we now have needed <= limit */ if (needed <= str->maxlen) return; /* got enough space already */ @@ -276,14 +316,14 @@ enlargeStringInfo(StringInfo str, int needed) newlen = 2 * newlen; /* - * Clamp to MaxAllocSize in case we went past it. Note we are assuming - * here that MaxAllocSize <= INT_MAX/2, else the above loop could - * overflow. We will still have newlen >= needed. + * Clamp to the limit in case we went past it. Note we are assuming here + * that limit <= INT_MAX/2, else the above loop could overflow. We will + * still have newlen >= needed. */ - if (newlen > (int) MaxAllocSize) - newlen = (int) MaxAllocSize; + if (newlen > limit) + newlen = limit; - str->data = (char *) repalloc(str->data, newlen); + str->data = (char *) repalloc_huge(str->data, (Size) newlen); str->maxlen = newlen; } diff --git a/src/include/lib/stringinfo.h b/src/include/lib/stringinfo.h index f644067..f65b12e 100644 --- a/src/include/lib/stringinfo.h +++ b/src/include/lib/stringinfo.h @@ -30,6 +30,8 @@ * cursor is initialized to zero by makeStringInfo or initStringInfo, * but is not otherwise touched by the stringinfo.c routines. * Some routines use it to scan through a StringInfo. + * long_ok whether this StringInfo can allocate more than MaxAllocSize + * bytes (but still up to 2GB). *------------------------- */ typedef struct StringInfoData @@ -38,6 +40,7 @@ typedef struct StringInfoData int len; int maxlen; int cursor; + bool long_ok; } StringInfoData; typedef StringInfoData *StringInfo; @@ -67,9 +70,12 @@ typedef StringInfoData *StringInfo; /*------------------------ * makeStringInfo - * Create an empty 'StringInfoData' & return a pointer to it. + * makeLongStringInfo + * Create an empty 'StringInfoData' & return a pointer to it. The former + * allows up to 1 GB in size, per palloc(); the latter allows longer strings. */ extern StringInfo makeStringInfo(void); +extern StringInfo makeLongStringInfo(void); /*------------------------ * initStringInfo @@ -79,6 +85,12 @@ extern StringInfo makeStringInfo(void); extern void initStringInfo(StringInfo str); /*------------------------ + * initLongStringInfo + * Same as initStringInfo but for larger strings (up to 2GB) + */ +extern void initLongStringInfo(StringInfo str); + +/*------------------------ * resetStringInfo * Clears the current content of the StringInfo, if any. The * StringInfo remains valid.