From 04fea919e59f3232fdc1e516ee0175a3da76f6ea Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 17 Mar 2026 16:47:22 -0400 Subject: [PATCH v1] Fix transient memory leakage in jsonpath evaluation. This patch reimplements JsonValueList to be more space-efficient and arranges for temporary JsonValueLists created during jsonpath evaluation to be freed when no longer needed, rather than being leaked till the end of the function evaluation cycle as before. The motivation is to prevent indefinite memory bloat while evaluating jsonpath expressions that traverse a lot of data. As an example, this query SELECT jsonb_path_query((SELECT jsonb_agg(i) FROM generate_series(1,10000) i), '$[*] ? (@ < $)'); formerly required about 6GB to execute, with the space required growing quadratically with the length of the input array. With this patch the memory consumption stays static. (The time required is still quadratic, but we can't do much about that: this path expression asks to compare each array element to each other one.) The bloat happens because we construct a JsonValueList containing all the array elements to represent the second occurrence of "$", and then just leak it after evaluating the filter expression for any one value generated from "$[*]". If I were implementing this functionality from scratch I'd probably try to avoid materializing that representation at all, but changing that now looks like more trouble than it's worth. This patch takes the more conservative approach of just making sure we free the list after we're done with it. The existing representation of JsonValueList is neither especially compact nor especially easy to free: it's a List containing pointers to separately-palloc'd JsonbValue structs. We could theoretically use list_free_deep, but it's not 100% clear that all the JsonbValues are always safe for us to free. In any case we are talking about a lot of palloc/pfree traffic if we keep it like this. This patch replaces that with what's essentially an expansible array of JsonbValues, so that even a long list requires relatively few palloc requests. Also, for the very common case that only one or two elements appear in the list, this representation uses *zero* pallocs: the elements can be kept in the on-the-stack base struct. Note that we are only interested in freeing the JsonbValue structs themselves. While many types of JsonbValue include pointers to external data such as strings or numerics, we expect that that data is part of the original jsonb input Datum(s) and need not (indeed cannot) be freed here. In this reimplementation, JsonValueListAppend() always copies the supplied JsonbValue struct into the JsonValueList data. This allows simplifying and regularizing many call sites that sometimes palloc'd JsonbValues and sometimes passed a local-variable JsonbValue. This should be a little faster than the old code thanks to reduction of palloc overhead, but I've not made any serious effort to quantify that. I do know it's over twice as fast as before for the example shown above, but that doesn't really represent typical usage. --- src/backend/utils/adt/jsonpath_exec.c | 536 ++++++++++++++++---------- 1 file changed, 330 insertions(+), 206 deletions(-) diff --git a/src/backend/utils/adt/jsonpath_exec.c b/src/backend/utils/adt/jsonpath_exec.c index 52ae0ba4cf7..4f08706363a 100644 --- a/src/backend/utils/adt/jsonpath_exec.c +++ b/src/backend/utils/adt/jsonpath_exec.c @@ -142,19 +142,46 @@ typedef enum JsonPathExecResult #define jperIsError(jper) ((jper) == jperError) /* - * List of jsonb values with shortcut for single-value list. + * List (or really array) of JsonbValues. This is the output representation + * of jsonpath evaluation. + * + * The initial or "base" chunk of a list is typically a local variable in + * a calling function. If we need more entries than will fit in the base + * chunk, we palloc more chunks. For notational simplicity, those are also + * treated as being of type JsonValueList, although they will have items[] + * arrays that are larger than BASE_JVL_ITEMS. + * + * Callers *must* initialize the base chunk with JsonValueListInit(). + * Typically they should free any extra chunks when done, using + * JsonValueListClear(), although some top-level functions skip that + * on the assumption that the caller's context will be reset soon. + * + * Note that most types of JsonbValue include pointers to external data, which + * will not be managed by the JsonValueList functions. We expect that such + * data is part of the input to the jsonpath operation, and the caller will + * see to it that it holds still for the duration of the operation. + * + * Most lists are short, though some can be quite long. So we set + * BASE_JVL_ITEMS small to conserve stack space, but grow the extra + * chunks aggressively. */ +#define BASE_JVL_ITEMS 2 /* number of items a base chunk holds */ +#define MIN_EXTRA_JVL_ITEMS 16 /* min number of items an extra chunk holds */ + typedef struct JsonValueList { - JsonbValue *singleton; - List *list; + int nitems; /* number of items stored in this chunk */ + int maxitems; /* allocated length of items[] */ + struct JsonValueList *next; /* => next chunk, if any */ + struct JsonValueList *last; /* => last chunk (only valid in base chunk) */ + JsonbValue items[BASE_JVL_ITEMS]; } JsonValueList; +/* State data for iterating through a JsonValueList */ typedef struct JsonValueListIterator { - JsonbValue *value; - List *list; - ListCell *next; + JsonValueList *chunk; /* current chunk of list */ + int nextitem; /* index of next value to return in chunk */ } JsonValueListIterator; /* Structures for JSON_TABLE execution */ @@ -270,7 +297,7 @@ static JsonPathExecResult executeItemUnwrapTargetArray(JsonPathExecContext *cxt, JsonValueList *found, bool unwrapElements); static JsonPathExecResult executeNextItem(JsonPathExecContext *cxt, JsonPathItem *cur, JsonPathItem *next, - JsonbValue *v, JsonValueList *found, bool copy); + JsonbValue *v, JsonValueList *found); static JsonPathExecResult executeItemOptUnwrapResult(JsonPathExecContext *cxt, JsonPathItem *jsp, JsonbValue *jb, bool unwrap, JsonValueList *found); static JsonPathExecResult executeItemOptUnwrapResultNoThrow(JsonPathExecContext *cxt, JsonPathItem *jsp, @@ -332,20 +359,19 @@ static JsonPathExecResult getArrayIndex(JsonPathExecContext *cxt, JsonPathItem *jsp, JsonbValue *jb, int32 *index); static JsonBaseObjectInfo setBaseObject(JsonPathExecContext *cxt, JsonbValue *jbv, int32 id); +static void JsonValueListInit(JsonValueList *jvl); static void JsonValueListClear(JsonValueList *jvl); -static void JsonValueListAppend(JsonValueList *jvl, JsonbValue *jbv); +static void JsonValueListAppend(JsonValueList *jvl, const JsonbValue *jbv); static int JsonValueListLength(const JsonValueList *jvl); static bool JsonValueListIsEmpty(JsonValueList *jvl); static JsonbValue *JsonValueListHead(JsonValueList *jvl); -static List *JsonValueListGetList(JsonValueList *jvl); -static void JsonValueListInitIterator(const JsonValueList *jvl, +static void JsonValueListInitIterator(JsonValueList *jvl, JsonValueListIterator *it); -static JsonbValue *JsonValueListNext(const JsonValueList *jvl, - JsonValueListIterator *it); +static JsonbValue *JsonValueListNext(JsonValueListIterator *it); static JsonbValue *JsonbInitBinary(JsonbValue *jbv, Jsonb *jb); static int JsonbType(JsonbValue *jb); static JsonbValue *getScalar(JsonbValue *scalar, enum jbvType type); -static JsonbValue *wrapItemsInArray(const JsonValueList *items); +static JsonbValue *wrapItemsInArray(JsonValueList *items); static int compareDatetime(Datum val1, Oid typid1, Datum val2, Oid typid2, bool useTz, bool *cast_error); static void checkTimezoneIsUsedForCast(bool useTz, const char *type1, @@ -456,9 +482,9 @@ jsonb_path_match_internal(FunctionCallInfo fcinfo, bool tz) { Jsonb *jb = PG_GETARG_JSONB_P(0); JsonPath *jp = PG_GETARG_JSONPATH_P(1); - JsonValueList found = {0}; Jsonb *vars = NULL; bool silent = true; + JsonValueList found; if (PG_NARGS() == 4) { @@ -466,6 +492,8 @@ jsonb_path_match_internal(FunctionCallInfo fcinfo, bool tz) silent = PG_GETARG_BOOL(3); } + JsonValueListInit(&found); + (void) executeJsonPath(jp, vars, getJsonPathVariableFromJsonb, countVariablesFromJsonb, jb, !silent, &found, tz); @@ -525,18 +553,17 @@ static Datum jsonb_path_query_internal(FunctionCallInfo fcinfo, bool tz) { FuncCallContext *funcctx; - List *found; + JsonValueListIterator *iter; JsonbValue *v; - ListCell *c; if (SRF_IS_FIRSTCALL()) { JsonPath *jp; Jsonb *jb; - MemoryContext oldcontext; Jsonb *vars; bool silent; - JsonValueList found = {0}; + MemoryContext oldcontext; + JsonValueList *found; funcctx = SRF_FIRSTCALL_INIT(); oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); @@ -546,26 +573,29 @@ jsonb_path_query_internal(FunctionCallInfo fcinfo, bool tz) vars = PG_GETARG_JSONB_P_COPY(2); silent = PG_GETARG_BOOL(3); + found = palloc_object(JsonValueList); + JsonValueListInit(found); + (void) executeJsonPath(jp, vars, getJsonPathVariableFromJsonb, countVariablesFromJsonb, - jb, !silent, &found, tz); + jb, !silent, found, tz); + + iter = palloc_object(JsonValueListIterator); + JsonValueListInitIterator(found, iter); - funcctx->user_fctx = JsonValueListGetList(&found); + funcctx->user_fctx = iter; MemoryContextSwitchTo(oldcontext); } funcctx = SRF_PERCALL_SETUP(); - found = funcctx->user_fctx; + iter = funcctx->user_fctx; - c = list_head(found); + v = JsonValueListNext(iter); - if (c == NULL) + if (v == NULL) SRF_RETURN_DONE(funcctx); - v = lfirst(c); - funcctx->user_fctx = list_delete_first(found); - SRF_RETURN_NEXT(funcctx, JsonbPGetDatum(JsonbValueToJsonb(v))); } @@ -591,9 +621,11 @@ jsonb_path_query_array_internal(FunctionCallInfo fcinfo, bool tz) { Jsonb *jb = PG_GETARG_JSONB_P(0); JsonPath *jp = PG_GETARG_JSONPATH_P(1); - JsonValueList found = {0}; Jsonb *vars = PG_GETARG_JSONB_P(2); bool silent = PG_GETARG_BOOL(3); + JsonValueList found; + + JsonValueListInit(&found); (void) executeJsonPath(jp, vars, getJsonPathVariableFromJsonb, countVariablesFromJsonb, @@ -624,15 +656,17 @@ jsonb_path_query_first_internal(FunctionCallInfo fcinfo, bool tz) { Jsonb *jb = PG_GETARG_JSONB_P(0); JsonPath *jp = PG_GETARG_JSONPATH_P(1); - JsonValueList found = {0}; Jsonb *vars = PG_GETARG_JSONB_P(2); bool silent = PG_GETARG_BOOL(3); + JsonValueList found; + + JsonValueListInit(&found); (void) executeJsonPath(jp, vars, getJsonPathVariableFromJsonb, countVariablesFromJsonb, jb, !silent, &found, tz); - if (JsonValueListLength(&found) >= 1) + if (!JsonValueListIsEmpty(&found)) PG_RETURN_JSONB_P(JsonbValueToJsonb(JsonValueListHead(&found))); else PG_RETURN_NULL(); @@ -710,14 +744,20 @@ executeJsonPath(JsonPath *path, void *vars, JsonPathGetVarCallback getVar, * In strict mode we must get a complete list of values to check that * there are no errors at all. */ - JsonValueList vals = {0}; + JsonValueList vals; + bool isempty; + + JsonValueListInit(&vals); res = executeItem(&cxt, &jsp, &jbv, &vals); + isempty = JsonValueListIsEmpty(&vals); + JsonValueListClear(&vals); + if (jperIsError(res)) return res; - return JsonValueListIsEmpty(&vals) ? jperNotFound : jperOk; + return isempty ? jperNotFound : jperOk; } res = executeItem(&cxt, &jsp, &jbv, result); @@ -761,8 +801,7 @@ executeItemOptUnwrapTarget(JsonPathExecContext *cxt, JsonPathItem *jsp, case jpiString: case jpiVariable: { - JsonbValue vbuf; - JsonbValue *v; + JsonbValue v; bool hasNext = jspGetNext(jsp, &elem); if (!hasNext && !found && jsp->type != jpiVariable) @@ -775,13 +814,11 @@ executeItemOptUnwrapTarget(JsonPathExecContext *cxt, JsonPathItem *jsp, break; } - v = hasNext ? &vbuf : palloc_object(JsonbValue); - baseObject = cxt->baseObject; - getJsonPathItem(cxt, jsp, v); + getJsonPathItem(cxt, jsp, &v); res = executeNextItem(cxt, jsp, &elem, - v, found, hasNext); + &v, found); cxt->baseObject = baseObject; } break; @@ -843,7 +880,7 @@ executeItemOptUnwrapTarget(JsonPathExecContext *cxt, JsonPathItem *jsp, jb, found, jspAutoUnwrap(cxt)); } else if (jspAutoWrap(cxt)) - res = executeNextItem(cxt, jsp, NULL, jb, found, true); + res = executeNextItem(cxt, jsp, NULL, jb, found); else if (!jspIgnoreStructuralErrors(cxt)) RETURN_ERROR(ereport(ERROR, (errcode(ERRCODE_SQL_JSON_ARRAY_NOT_FOUND), @@ -932,12 +969,10 @@ executeItemOptUnwrapTarget(JsonPathExecContext *cxt, JsonPathItem *jsp, for (index = index_from; index <= index_to; index++) { JsonbValue *v; - bool copy; if (singleton) { v = jb; - copy = true; } else { @@ -946,15 +981,12 @@ executeItemOptUnwrapTarget(JsonPathExecContext *cxt, JsonPathItem *jsp, if (v == NULL) continue; - - copy = false; } if (!hasNext && !found) return jperOk; - res = executeNextItem(cxt, jsp, &elem, v, found, - copy); + res = executeNextItem(cxt, jsp, &elem, v, found); if (jperIsError(res)) break; @@ -992,7 +1024,7 @@ executeItemOptUnwrapTarget(JsonPathExecContext *cxt, JsonPathItem *jsp, savedIgnoreStructuralErrors = cxt->ignoreStructuralErrors; cxt->ignoreStructuralErrors = true; res = executeNextItem(cxt, jsp, &elem, - jb, found, true); + jb, found); cxt->ignoreStructuralErrors = savedIgnoreStructuralErrors; if (res == jperOk && !found) @@ -1025,11 +1057,8 @@ executeItemOptUnwrapTarget(JsonPathExecContext *cxt, JsonPathItem *jsp, if (v != NULL) { res = executeNextItem(cxt, jsp, NULL, - v, found, false); - - /* free value if it was not added to found list */ - if (jspHasNext(jsp) || !found) - pfree(v); + v, found); + pfree(v); } else if (!jspIgnoreStructuralErrors(cxt)) { @@ -1057,14 +1086,13 @@ executeItemOptUnwrapTarget(JsonPathExecContext *cxt, JsonPathItem *jsp, break; case jpiCurrent: - res = executeNextItem(cxt, jsp, NULL, cxt->current, - found, true); + res = executeNextItem(cxt, jsp, NULL, cxt->current, found); break; case jpiRoot: jb = cxt->root; baseObject = setBaseObject(cxt, jb, 0); - res = executeNextItem(cxt, jsp, NULL, jb, found, true); + res = executeNextItem(cxt, jsp, NULL, jb, found); cxt->baseObject = baseObject; break; @@ -1082,26 +1110,26 @@ executeItemOptUnwrapTarget(JsonPathExecContext *cxt, JsonPathItem *jsp, res = jperNotFound; else res = executeNextItem(cxt, jsp, NULL, - jb, found, true); + jb, found); break; } case jpiType: { - JsonbValue *jbv = palloc_object(JsonbValue); + JsonbValue jbv; - jbv->type = jbvString; - jbv->val.string.val = pstrdup(JsonbTypeName(jb)); - jbv->val.string.len = strlen(jbv->val.string.val); + jbv.type = jbvString; + jbv.val.string.val = pstrdup(JsonbTypeName(jb)); + jbv.val.string.len = strlen(jbv.val.string.val); - res = executeNextItem(cxt, jsp, NULL, jbv, - found, false); + res = executeNextItem(cxt, jsp, NULL, &jbv, found); } break; case jpiSize: { int size = JsonbArraySize(jb); + JsonbValue jbv; if (size < 0) { @@ -1118,12 +1146,10 @@ executeItemOptUnwrapTarget(JsonPathExecContext *cxt, JsonPathItem *jsp, size = 1; } - jb = palloc_object(JsonbValue); - - jb->type = jbvNumeric; - jb->val.numeric = int64_to_numeric(size); + jbv.type = jbvNumeric; + jbv.val.numeric = int64_to_numeric(size); - res = executeNextItem(cxt, jsp, NULL, jb, found, false); + res = executeNextItem(cxt, jsp, NULL, &jbv, found); } break; @@ -1210,7 +1236,7 @@ executeItemOptUnwrapTarget(JsonPathExecContext *cxt, JsonPathItem *jsp, errmsg("jsonpath item method .%s() can only be applied to a string or numeric value", jspOperationName(jsp->type))))); - res = executeNextItem(cxt, jsp, NULL, jb, found, true); + res = executeNextItem(cxt, jsp, NULL, jb, found); } break; @@ -1233,8 +1259,7 @@ executeItemOptUnwrapTarget(JsonPathExecContext *cxt, JsonPathItem *jsp, case jpiLast: { - JsonbValue tmpjbv; - JsonbValue *lastjbv; + JsonbValue jbv; int last; bool hasNext = jspGetNext(jsp, &elem); @@ -1249,13 +1274,11 @@ executeItemOptUnwrapTarget(JsonPathExecContext *cxt, JsonPathItem *jsp, last = cxt->innermostArraySize - 1; - lastjbv = hasNext ? &tmpjbv : palloc_object(JsonbValue); - - lastjbv->type = jbvNumeric; - lastjbv->val.numeric = int64_to_numeric(last); + jbv.type = jbvNumeric; + jbv.val.numeric = int64_to_numeric(last); res = executeNextItem(cxt, jsp, &elem, - lastjbv, found, hasNext); + &jbv, found); } break; @@ -1314,12 +1337,11 @@ executeItemOptUnwrapTarget(JsonPathExecContext *cxt, JsonPathItem *jsp, errmsg("jsonpath item method .%s() can only be applied to a string or numeric value", jspOperationName(jsp->type))))); - jb = &jbv; - jb->type = jbvNumeric; - jb->val.numeric = DatumGetNumeric(DirectFunctionCall1(int8_numeric, + jbv.type = jbvNumeric; + jbv.val.numeric = DatumGetNumeric(DirectFunctionCall1(int8_numeric, datum)); - res = executeNextItem(cxt, jsp, NULL, jb, found, true); + res = executeNextItem(cxt, jsp, NULL, &jbv, found); } break; @@ -1387,11 +1409,10 @@ executeItemOptUnwrapTarget(JsonPathExecContext *cxt, JsonPathItem *jsp, errmsg("jsonpath item method .%s() can only be applied to a boolean, string, or numeric value", jspOperationName(jsp->type))))); - jb = &jbv; - jb->type = jbvBool; - jb->val.boolean = bval; + jbv.type = jbvBool; + jbv.val.boolean = bval; - res = executeNextItem(cxt, jsp, NULL, jb, found, true); + res = executeNextItem(cxt, jsp, NULL, &jbv, found); } break; @@ -1532,11 +1553,10 @@ executeItemOptUnwrapTarget(JsonPathExecContext *cxt, JsonPathItem *jsp, pfree(arrtypmod); } - jb = &jbv; - jb->type = jbvNumeric; - jb->val.numeric = num; + jbv.type = jbvNumeric; + jbv.val.numeric = num; - res = executeNextItem(cxt, jsp, NULL, jb, found, true); + res = executeNextItem(cxt, jsp, NULL, &jbv, found); } break; @@ -1594,12 +1614,11 @@ executeItemOptUnwrapTarget(JsonPathExecContext *cxt, JsonPathItem *jsp, errmsg("jsonpath item method .%s() can only be applied to a string or numeric value", jspOperationName(jsp->type))))); - jb = &jbv; - jb->type = jbvNumeric; - jb->val.numeric = DatumGetNumeric(DirectFunctionCall1(int4_numeric, + jbv.type = jbvNumeric; + jbv.val.numeric = DatumGetNumeric(DirectFunctionCall1(int4_numeric, datum)); - res = executeNextItem(cxt, jsp, NULL, jb, found, true); + res = executeNextItem(cxt, jsp, NULL, &jbv, found); } break; @@ -1651,13 +1670,12 @@ executeItemOptUnwrapTarget(JsonPathExecContext *cxt, JsonPathItem *jsp, break; } - jb = &jbv; Assert(tmp != NULL); /* We must have set tmp above */ - jb->val.string.val = tmp; - jb->val.string.len = strlen(jb->val.string.val); - jb->type = jbvString; + jbv.val.string.val = tmp; + jbv.val.string.len = strlen(jbv.val.string.val); + jbv.type = jbvString; - res = executeNextItem(cxt, jsp, NULL, jb, found, true); + res = executeNextItem(cxt, jsp, NULL, &jbv, found); } break; @@ -1694,7 +1712,7 @@ executeItemUnwrapTargetArray(JsonPathExecContext *cxt, JsonPathItem *jsp, static JsonPathExecResult executeNextItem(JsonPathExecContext *cxt, JsonPathItem *cur, JsonPathItem *next, - JsonbValue *v, JsonValueList *found, bool copy) + JsonbValue *v, JsonValueList *found) { JsonPathItem elem; bool hasNext; @@ -1713,7 +1731,7 @@ executeNextItem(JsonPathExecContext *cxt, return executeItem(cxt, next, v, found); if (found) - JsonValueListAppend(found, copy ? copyJsonbValue(v) : v); + JsonValueListAppend(found, v); return jperOk; } @@ -1729,16 +1747,23 @@ executeItemOptUnwrapResult(JsonPathExecContext *cxt, JsonPathItem *jsp, { if (unwrap && jspAutoUnwrap(cxt)) { - JsonValueList seq = {0}; + JsonValueList seq; JsonValueListIterator it; - JsonPathExecResult res = executeItem(cxt, jsp, jb, &seq); + JsonPathExecResult res; JsonbValue *item; + JsonValueListInit(&seq); + + res = executeItem(cxt, jsp, jb, &seq); + if (jperIsError(res)) + { + JsonValueListClear(&seq); return res; + } JsonValueListInitIterator(&seq, &it); - while ((item = JsonValueListNext(&seq, &it))) + while ((item = JsonValueListNext(&it))) { Assert(item->type != jbvArray); @@ -1748,6 +1773,8 @@ executeItemOptUnwrapResult(JsonPathExecContext *cxt, JsonPathItem *jsp, JsonValueListAppend(found, item); } + JsonValueListClear(&seq); + return jperOk; } @@ -1878,15 +1905,22 @@ executeBoolItem(JsonPathExecContext *cxt, JsonPathItem *jsp, * In strict mode we must get a complete list of values to * check that there are no errors at all. */ - JsonValueList vals = {0}; - JsonPathExecResult res = - executeItemOptUnwrapResultNoThrow(cxt, &larg, jb, - false, &vals); + JsonValueList vals; + JsonPathExecResult res; + bool isempty; + + JsonValueListInit(&vals); + + res = executeItemOptUnwrapResultNoThrow(cxt, &larg, jb, + false, &vals); + + isempty = JsonValueListIsEmpty(&vals); + JsonValueListClear(&vals); if (jperIsError(res)) return jpbUnknown; - return JsonValueListIsEmpty(&vals) ? jpbFalse : jpbTrue; + return isempty ? jpbFalse : jpbTrue; } else { @@ -1988,7 +2022,7 @@ executeAnyItem(JsonPathExecContext *cxt, JsonPathItem *jsp, JsonbContainer *jbc, break; } else if (found) - JsonValueListAppend(found, copyJsonbValue(&v)); + JsonValueListAppend(found, &v); else return jperOk; } @@ -2030,16 +2064,22 @@ executePredicate(JsonPathExecContext *cxt, JsonPathItem *pred, { JsonPathExecResult res; JsonValueListIterator lseqit; - JsonValueList lseq = {0}; - JsonValueList rseq = {0}; + JsonValueList lseq; + JsonValueList rseq; JsonbValue *lval; bool error = false; bool found = false; + JsonValueListInit(&lseq); + JsonValueListInit(&rseq); + /* Left argument is always auto-unwrapped. */ res = executeItemOptUnwrapResultNoThrow(cxt, larg, jb, true, &lseq); if (jperIsError(res)) - return jpbUnknown; + { + error = true; + goto exit; + } if (rarg) { @@ -2047,11 +2087,14 @@ executePredicate(JsonPathExecContext *cxt, JsonPathItem *pred, res = executeItemOptUnwrapResultNoThrow(cxt, rarg, jb, unwrapRightArg, &rseq); if (jperIsError(res)) - return jpbUnknown; + { + error = true; + goto exit; + } } JsonValueListInitIterator(&lseq, &lseqit); - while ((lval = JsonValueListNext(&lseq, &lseqit))) + while ((lval = JsonValueListNext(&lseqit))) { JsonValueListIterator rseqit; JsonbValue *rval; @@ -2059,7 +2102,7 @@ executePredicate(JsonPathExecContext *cxt, JsonPathItem *pred, JsonValueListInitIterator(&rseq, &rseqit); if (rarg) - rval = JsonValueListNext(&rseq, &rseqit); + rval = JsonValueListNext(&rseqit); else rval = NULL; @@ -2070,25 +2113,30 @@ executePredicate(JsonPathExecContext *cxt, JsonPathItem *pred, if (res == jpbUnknown) { - if (jspStrictAbsenceOfErrors(cxt)) - return jpbUnknown; - error = true; + if (jspStrictAbsenceOfErrors(cxt)) + { + found = false; /* return unknown, not success */ + goto exit; + } } else if (res == jpbTrue) { - if (!jspStrictAbsenceOfErrors(cxt)) - return jpbTrue; - found = true; + if (!jspStrictAbsenceOfErrors(cxt)) + goto exit; } first = false; if (rarg) - rval = JsonValueListNext(&rseq, &rseqit); + rval = JsonValueListNext(&rseqit); } } +exit: + JsonValueListClear(&lseq); + JsonValueListClear(&rseq); + if (found) /* possible only in strict mode */ return jpbTrue; @@ -2109,12 +2157,16 @@ executeBinaryArithmExpr(JsonPathExecContext *cxt, JsonPathItem *jsp, { JsonPathExecResult jper; JsonPathItem elem; - JsonValueList lseq = {0}; - JsonValueList rseq = {0}; + JsonValueList lseq; + JsonValueList rseq; JsonbValue *lval; JsonbValue *rval; + JsonbValue resval; Numeric res; + JsonValueListInit(&lseq); + JsonValueListInit(&rseq); + jspGetLeftArg(jsp, &elem); /* @@ -2123,27 +2175,43 @@ executeBinaryArithmExpr(JsonPathExecContext *cxt, JsonPathItem *jsp, */ jper = executeItemOptUnwrapResult(cxt, &elem, jb, true, &lseq); if (jperIsError(jper)) + { + JsonValueListClear(&lseq); + JsonValueListClear(&rseq); return jper; + } jspGetRightArg(jsp, &elem); jper = executeItemOptUnwrapResult(cxt, &elem, jb, true, &rseq); if (jperIsError(jper)) + { + JsonValueListClear(&lseq); + JsonValueListClear(&rseq); return jper; + } if (JsonValueListLength(&lseq) != 1 || !(lval = getScalar(JsonValueListHead(&lseq), jbvNumeric))) + { + JsonValueListClear(&lseq); + JsonValueListClear(&rseq); RETURN_ERROR(ereport(ERROR, (errcode(ERRCODE_SINGLETON_SQL_JSON_ITEM_REQUIRED), errmsg("left operand of jsonpath operator %s is not a single numeric value", jspOperationName(jsp->type))))); + } if (JsonValueListLength(&rseq) != 1 || !(rval = getScalar(JsonValueListHead(&rseq), jbvNumeric))) + { + JsonValueListClear(&lseq); + JsonValueListClear(&rseq); RETURN_ERROR(ereport(ERROR, (errcode(ERRCODE_SINGLETON_SQL_JSON_ITEM_REQUIRED), errmsg("right operand of jsonpath operator %s is not a single numeric value", jspOperationName(jsp->type))))); + } if (jspThrowErrors(cxt)) { @@ -2156,17 +2224,23 @@ executeBinaryArithmExpr(JsonPathExecContext *cxt, JsonPathItem *jsp, res = func(lval->val.numeric, rval->val.numeric, (Node *) &escontext); if (escontext.error_occurred) + { + JsonValueListClear(&lseq); + JsonValueListClear(&rseq); return jperError; + } } + JsonValueListClear(&lseq); + JsonValueListClear(&rseq); + if (!jspGetNext(jsp, &elem) && !found) return jperOk; - lval = palloc_object(JsonbValue); - lval->type = jbvNumeric; - lval->val.numeric = res; + resval.type = jbvNumeric; + resval.val.numeric = res; - return executeNextItem(cxt, jsp, &elem, lval, found, false); + return executeNextItem(cxt, jsp, &elem, &resval, found); } /* @@ -2180,34 +2254,40 @@ executeUnaryArithmExpr(JsonPathExecContext *cxt, JsonPathItem *jsp, JsonPathExecResult jper; JsonPathExecResult jper2; JsonPathItem elem; - JsonValueList seq = {0}; + JsonValueList seq; JsonValueListIterator it; JsonbValue *val; bool hasNext; + JsonValueListInit(&seq); + jspGetArg(jsp, &elem); jper = executeItemOptUnwrapResult(cxt, &elem, jb, true, &seq); if (jperIsError(jper)) - return jper; + goto exit; jper = jperNotFound; hasNext = jspGetNext(jsp, &elem); JsonValueListInitIterator(&seq, &it); - while ((val = JsonValueListNext(&seq, &it))) + while ((val = JsonValueListNext(&it))) { if ((val = getScalar(val, jbvNumeric))) { if (!found && !hasNext) - return jperOk; + { + jper = jperOk; + goto exit; + } } else { if (!found && !hasNext) continue; /* skip non-numerics processing */ + JsonValueListClear(&seq); RETURN_ERROR(ereport(ERROR, (errcode(ERRCODE_SQL_JSON_NUMBER_NOT_FOUND), errmsg("operand of unary jsonpath operator %s is not a numeric value", @@ -2219,19 +2299,25 @@ executeUnaryArithmExpr(JsonPathExecContext *cxt, JsonPathItem *jsp, DatumGetNumeric(DirectFunctionCall1(func, NumericGetDatum(val->val.numeric))); - jper2 = executeNextItem(cxt, jsp, &elem, val, found, false); + jper2 = executeNextItem(cxt, jsp, &elem, val, found); if (jperIsError(jper2)) - return jper2; + { + jper = jper2; + goto exit; + } if (jper2 == jperOk) { - if (!found) - return jperOk; jper = jperOk; + if (!found) + goto exit; } } +exit: + JsonValueListClear(&seq); + return jper; } @@ -2302,6 +2388,7 @@ executeNumericItemMethod(JsonPathExecContext *cxt, JsonPathItem *jsp, { JsonPathItem next; Datum datum; + JsonbValue jbv; if (unwrap && JsonbType(jb) == jbvArray) return executeItemUnwrapTargetArray(cxt, jsp, jb, found, false); @@ -2317,11 +2404,10 @@ executeNumericItemMethod(JsonPathExecContext *cxt, JsonPathItem *jsp, if (!jspGetNext(jsp, &next) && !found) return jperOk; - jb = palloc_object(JsonbValue); - jb->type = jbvNumeric; - jb->val.numeric = DatumGetNumeric(datum); + jbv.type = jbvNumeric; + jbv.val.numeric = DatumGetNumeric(datum); - return executeNextItem(cxt, jsp, &next, jb, found, false); + return executeNextItem(cxt, jsp, &next, &jbv, found); } /* @@ -2340,7 +2426,7 @@ static JsonPathExecResult executeDateTimeMethod(JsonPathExecContext *cxt, JsonPathItem *jsp, JsonbValue *jb, JsonValueList *found) { - JsonbValue jbvbuf; + JsonbValue jbv; Datum value; text *datetime; Oid collid; @@ -2783,15 +2869,13 @@ executeDateTimeMethod(JsonPathExecContext *cxt, JsonPathItem *jsp, if (!hasNext && !found) return res; - jb = hasNext ? &jbvbuf : palloc_object(JsonbValue); - - jb->type = jbvDatetime; - jb->val.datetime.value = value; - jb->val.datetime.typid = typid; - jb->val.datetime.typmod = typmod; - jb->val.datetime.tz = tz; + jbv.type = jbvDatetime; + jbv.val.datetime.value = value; + jbv.val.datetime.typid = typid; + jbv.val.datetime.typmod = typmod; + jbv.val.datetime.tz = tz; - return executeNextItem(cxt, jsp, &elem, jb, found, hasNext); + return executeNextItem(cxt, jsp, &elem, &jbv, found); } /* @@ -2909,7 +2993,7 @@ executeKeyValueMethod(JsonPathExecContext *cxt, JsonPathItem *jsp, baseObject = setBaseObject(cxt, &obj, cxt->lastGeneratedObjectId++); - res = executeNextItem(cxt, jsp, &next, &obj, found, true); + res = executeNextItem(cxt, jsp, &next, &obj, found); cxt->baseObject = baseObject; @@ -2947,7 +3031,7 @@ appendBoolResult(JsonPathExecContext *cxt, JsonPathItem *jsp, jbv.val.boolean = res == jpbTrue; } - return executeNextItem(cxt, jsp, &next, &jbv, found, true); + return executeNextItem(cxt, jsp, &next, &jbv, found); } /* @@ -3461,19 +3545,29 @@ getArrayIndex(JsonPathExecContext *cxt, JsonPathItem *jsp, JsonbValue *jb, int32 *index) { JsonbValue *jbv; - JsonValueList found = {0}; - JsonPathExecResult res = executeItem(cxt, jsp, jb, &found); + JsonValueList found; + JsonPathExecResult res; Datum numeric_index; ErrorSaveContext escontext = {T_ErrorSaveContext}; + JsonValueListInit(&found); + + res = executeItem(cxt, jsp, jb, &found); + if (jperIsError(res)) + { + JsonValueListClear(&found); return res; + } if (JsonValueListLength(&found) != 1 || !(jbv = getScalar(JsonValueListHead(&found), jbvNumeric))) + { + JsonValueListClear(&found); RETURN_ERROR(ereport(ERROR, (errcode(ERRCODE_INVALID_SQL_JSON_SUBSCRIPT), errmsg("jsonpath array subscript is not a single numeric value")))); + } numeric_index = DirectFunctionCall2(numeric_trunc, NumericGetDatum(jbv->val.numeric), @@ -3482,6 +3576,8 @@ getArrayIndex(JsonPathExecContext *cxt, JsonPathItem *jsp, JsonbValue *jb, *index = numeric_int4_safe(DatumGetNumeric(numeric_index), (Node *) &escontext); + JsonValueListClear(&found); + if (escontext.error_occurred) RETURN_ERROR(ereport(ERROR, (errcode(ERRCODE_INVALID_SQL_JSON_SUBSCRIPT), @@ -3503,96 +3599,119 @@ setBaseObject(JsonPathExecContext *cxt, JsonbValue *jbv, int32 id) return baseObject; } +/* + * JsonValueList support functions + */ + +static void +JsonValueListInit(JsonValueList *jvl) +{ + jvl->nitems = 0; + jvl->maxitems = BASE_JVL_ITEMS; + jvl->next = NULL; + jvl->last = jvl; +} + static void JsonValueListClear(JsonValueList *jvl) { - jvl->singleton = NULL; - jvl->list = NIL; + JsonValueList *nxt; + + /* Release any extra chunks */ + for (JsonValueList *chunk = jvl->next; chunk != NULL; chunk = nxt) + { + nxt = chunk->next; + pfree(chunk); + } + /* ... and reset to empty */ + jvl->nitems = 0; + Assert(jvl->maxitems == BASE_JVL_ITEMS); + jvl->next = NULL; + jvl->last = jvl; } static void -JsonValueListAppend(JsonValueList *jvl, JsonbValue *jbv) +JsonValueListAppend(JsonValueList *jvl, const JsonbValue *jbv) { - if (jvl->singleton) + JsonValueList *last = jvl->last; + + if (last->nitems < last->maxitems) { - jvl->list = list_make2(jvl->singleton, jbv); - jvl->singleton = NULL; + /* there's still room in the last existing chunk */ + last->items[last->nitems] = *jbv; + last->nitems++; } - else if (!jvl->list) - jvl->singleton = jbv; else - jvl->list = lappend(jvl->list, jbv); + { + /* need a new last chunk */ + JsonValueList *nxt; + int nxtsize; + + nxtsize = last->maxitems * 2; /* double the size with each chunk */ + nxtsize = Max(nxtsize, MIN_EXTRA_JVL_ITEMS); /* but at least this */ + nxt = palloc(offsetof(JsonValueList, items) + + nxtsize * sizeof(JsonbValue)); + nxt->nitems = 1; + nxt->maxitems = nxtsize; + nxt->next = NULL; + nxt->items[0] = *jbv; + last->next = nxt; + jvl->last = nxt; + } } static int JsonValueListLength(const JsonValueList *jvl) { - return jvl->singleton ? 1 : list_length(jvl->list); + int len = 0; + + for (const JsonValueList *chunk = jvl; chunk != NULL; chunk = chunk->next) + len += chunk->nitems; + return len; } static bool JsonValueListIsEmpty(JsonValueList *jvl) { - return !jvl->singleton && (jvl->list == NIL); + return (jvl->nitems == 0); } static JsonbValue * JsonValueListHead(JsonValueList *jvl) { - return jvl->singleton ? jvl->singleton : linitial(jvl->list); + Assert(jvl->nitems > 0); + return &jvl->items[0]; } -static List * -JsonValueListGetList(JsonValueList *jvl) -{ - if (jvl->singleton) - return list_make1(jvl->singleton); - - return jvl->list; -} +/* + * JsonValueListIterator functions + */ static void -JsonValueListInitIterator(const JsonValueList *jvl, JsonValueListIterator *it) +JsonValueListInitIterator(JsonValueList *jvl, JsonValueListIterator *it) { - if (jvl->singleton) - { - it->value = jvl->singleton; - it->list = NIL; - it->next = NULL; - } - else if (jvl->list != NIL) - { - it->value = (JsonbValue *) linitial(jvl->list); - it->list = jvl->list; - it->next = list_second_cell(jvl->list); - } - else - { - it->value = NULL; - it->list = NIL; - it->next = NULL; - } + it->chunk = jvl; + it->nextitem = 0; } /* * Get the next item from the sequence advancing iterator. + * Returns NULL if no more items. */ static JsonbValue * -JsonValueListNext(const JsonValueList *jvl, JsonValueListIterator *it) +JsonValueListNext(JsonValueListIterator *it) { - JsonbValue *result = it->value; - - if (it->next) - { - it->value = lfirst(it->next); - it->next = lnext(it->list, it->next); - } - else + if (it->chunk == NULL) + return NULL; + if (it->nextitem >= it->chunk->nitems) { - it->value = NULL; + it->chunk = it->chunk->next; + if (it->chunk == NULL) + return NULL; + it->nextitem = 0; + Assert(it->chunk->nitems > 0); } - - return result; + return &it->chunk->items[it->nextitem++]; } /* @@ -3647,7 +3766,7 @@ getScalar(JsonbValue *scalar, enum jbvType type) /* Construct a JSON array from the item list */ static JsonbValue * -wrapItemsInArray(const JsonValueList *items) +wrapItemsInArray(JsonValueList *items) { JsonbInState ps = {0}; JsonValueListIterator it; @@ -3656,7 +3775,7 @@ wrapItemsInArray(const JsonValueList *items) pushJsonbValue(&ps, WJB_BEGIN_ARRAY, NULL); JsonValueListInitIterator(items, &it); - while ((jbv = JsonValueListNext(items, &it))) + while ((jbv = JsonValueListNext(&it))) pushJsonbValue(&ps, WJB_ELEM, jbv); pushJsonbValue(&ps, WJB_END_ARRAY, NULL); @@ -3917,10 +4036,12 @@ JsonPathQuery(Datum jb, JsonPath *jp, JsonWrapper wrapper, bool *empty, { JsonbValue *singleton; bool wrap; - JsonValueList found = {0}; + JsonValueList found; JsonPathExecResult res; int count; + JsonValueListInit(&found); + res = executeJsonPath(jp, vars, GetJsonPathVar, CountJsonPathVars, DatumGetJsonbP(jb), !error, &found, true); @@ -4009,10 +4130,12 @@ JsonPathValue(Datum jb, JsonPath *jp, bool *empty, bool *error, List *vars, const char *column_name) { JsonbValue *res; - JsonValueList found = {0}; + JsonValueList found; JsonPathExecResult jper PG_USED_FOR_ASSERTS_ONLY; int count; + JsonValueListInit(&found); + jper = executeJsonPath(jp, vars, GetJsonPathVar, CountJsonPathVars, DatumGetJsonbP(jb), !error, &found, true); @@ -4053,7 +4176,7 @@ JsonPathValue(Datum jb, JsonPath *jp, bool *empty, bool *error, List *vars, errmsg("JSON path expression in JSON_VALUE must return single scalar item"))); } - res = JsonValueListHead(&found); + res = copyJsonbValue(JsonValueListHead(&found)); if (res->type == jbvBinary && JsonContainerIsScalar(res->val.binary.data)) JsonbExtractScalar(res->val.binary.data, res); @@ -4200,6 +4323,7 @@ JsonTableInitPlan(JsonTableExecContext *cxt, JsonTablePlan *plan, planstate->plan = plan; planstate->parent = parentstate; + JsonValueListInit(&planstate->found); if (IsA(plan, JsonTablePathScan)) { @@ -4335,7 +4459,7 @@ JsonTablePlanScanNextRow(JsonTablePlanState *planstate) } /* Fetch new row from the list of found values to set as active. */ - jbv = JsonValueListNext(&planstate->found, &planstate->iter); + jbv = JsonValueListNext(&planstate->iter); /* End of list? */ if (jbv == NULL) -- 2.43.7