From d76c16947925d89ac16b7f5c643a85b895a0efa9 Mon Sep 17 00:00:00 2001 From: Zsolt Parragi Date: Wed, 8 Jul 2026 22:14:19 +0000 Subject: [PATCH 3/8] Add JSON5 unquoted object key support to the JSON parser When json5 is set, accept identifier object keys (letter/_/$ start, alphanumerics after) and the reserved words true/false/null as keys, in both parser paths. --- src/common/jsonapi.c | 145 ++++++++++++++++-- src/include/common/jsonapi.h | 2 + .../modules/test_json_parser/json5_keys.json5 | 12 ++ .../modules/test_json_parser/json5_keys.out | 29 ++++ .../test_json_parser/t/005_test_json5.pl | 11 +- 5 files changed, 180 insertions(+), 19 deletions(-) create mode 100644 src/test/modules/test_json_parser/json5_keys.json5 create mode 100644 src/test/modules/test_json_parser/json5_keys.out diff --git a/src/common/jsonapi.c b/src/common/jsonapi.c index 7e94b4c4200..e8634a749ea 100644 --- a/src/common/jsonapi.c +++ b/src/common/jsonapi.c @@ -661,6 +661,33 @@ have_prediction(JsonParserStack *pstack) return pstack->pred_index > 0; } +/* reserved words that json5 accepts as unquoted object keys */ +static inline bool +json5_keyword_key(const JsonLexContext *lex, JsonTokenType tok) +{ + return tok == JSON_TOKEN_TRUE || tok == JSON_TOKEN_FALSE || + tok == JSON_TOKEN_NULL; +} + +/* + * Copy the current token verbatim, for a json5 keyword used as an unquoted + * object key (the lexer only fills strval for real strings). Returns NULL + * on allocation failure. + */ +static char * +json5_dup_reserved_key(JsonLexContext *lex) +{ + size_t klen = lex->token_terminator - lex->token_start; + char *fname = ALLOC(klen + 1); + + if (fname != NULL) + { + memcpy(fname, lex->token_start, klen); + fname[klen] = '\0'; + } + return fname; +} + /* * json5 trailing comma support: discard predictions for an element that * will never arrive, stopping at the enclosing close bracket/brace or the @@ -967,8 +994,16 @@ pg_parse_json_incremental(JsonLexContext *lex, /* * these first two branches are the guts of the Table Driven method + * + * In json5 mode a reserved word (true/false/null) also matches a + * predicted key: key position is recognizable by the COLON prediction + * that directly follows it. */ - if (top == tok) + if (top == tok || + (lex->json5 && + top == JSON_TOKEN_STRING && + json5_keyword_key(lex, tok) && + have_prediction(pstack) && next_prediction(pstack) == JSON_TOKEN_COLON)) { /* * tok can only be a terminal symbol, so top must be too. the @@ -984,12 +1019,23 @@ pg_parse_json_incremental(JsonLexContext *lex, tok = lex_peek(lex); } } - else if (IS_NT(top) && (entry = td_parser_table[OFS(top)][tok]).prod != NULL) + else if (IS_NT(top) && + (((entry = td_parser_table[OFS(top)][tok]).prod != NULL && + !(top == JSON_NT_JSON && tok == JSON_TOKEN_STRING && + lex->token_is_identifier)) || + (lex->json5 && + (top == JSON_NT_KEY_PAIRS || top == JSON_NT_MORE_KEY_PAIRS) && + json5_keyword_key(lex, tok) && + (entry = td_parser_table[OFS(top)][JSON_TOKEN_STRING]).prod != NULL))) { /* * the token is in the director set for a production of the * non-terminal at the top of the stack, so push the reversed RHS * of the production onto the stack. + * + * Two json5 deviations above: an unquoted identifier (lexed as a + * string) must not be taken as a value, and a reserved word + * (true/false/null) doubles as a key. */ push_prediction(pstack, entry); } @@ -1079,9 +1125,18 @@ pg_parse_json_incremental(JsonLexContext *lex, if ((ostart != NULL || oend != NULL) && lex->need_escapes) { - fname = STRDUP(lex->strval->data); - if (fname == NULL) - return JSON_OUT_OF_MEMORY; + if (lex->json5 && json5_keyword_key(lex, tok)) + { + fname = json5_dup_reserved_key(lex); + if (fname == NULL) + return JSON_OUT_OF_MEMORY; + } + else + { + fname = STRDUP(lex->strval->data); + if (fname == NULL) + return JSON_OUT_OF_MEMORY; + } } set_fname(lex, fname); } @@ -1322,6 +1377,10 @@ parse_scalar(JsonLexContext *lex, const JsonSemAction *sem) tok != JSON_TOKEN_NULL) return report_parse_error(JSON_PARSE_VALUE, lex); + /* json5 unquoted identifiers are lexed as strings but can't be values */ + if (tok == JSON_TOKEN_STRING && lex->token_is_identifier) + return report_parse_error(JSON_PARSE_VALUE, lex); + /* if no semantic function, just consume the token */ if (sfunc == NULL) return json_lex(lex); @@ -1384,15 +1443,27 @@ parse_object_field(JsonLexContext *lex, const JsonSemAction *sem) JsonTokenType tok; JsonParseErrorType result; - if (lex_peek(lex) != JSON_TOKEN_STRING) - return report_parse_error(JSON_PARSE_STRING, lex); - if ((ostart != NULL || oend != NULL) && lex->need_escapes) + if (lex_peek(lex) == JSON_TOKEN_STRING) { - /* fname is NULL if need_escapes is false */ - fname = STRDUP(lex->strval->data); - if (fname == NULL) - return JSON_OUT_OF_MEMORY; + if ((ostart != NULL || oend != NULL) && lex->need_escapes) + { + /* fname is NULL if need_escapes is false */ + fname = STRDUP(lex->strval->data); + if (fname == NULL) + return JSON_OUT_OF_MEMORY; + } + } + else if (lex->json5 && json5_keyword_key(lex, lex_peek(lex))) + { + if ((ostart != NULL || oend != NULL) && lex->need_escapes) + { + fname = json5_dup_reserved_key(lex); + if (fname == NULL) + return JSON_OUT_OF_MEMORY; + } } + else + return report_parse_error(JSON_PARSE_STRING, lex); result = json_lex(lex); if (result != JSON_SUCCESS) { @@ -1488,6 +1559,15 @@ parse_object(JsonLexContext *lex, const JsonSemAction *sem) tok = lex_peek(lex); switch (tok) { + case JSON_TOKEN_TRUE: + case JSON_TOKEN_FALSE: + case JSON_TOKEN_NULL: + if (!lex->json5) + { + result = report_parse_error(JSON_PARSE_OBJECT_START, lex); + break; + } + pg_fallthrough; case JSON_TOKEN_STRING: result = parse_object_field(lex, sem); while (result == JSON_SUCCESS && lex_peek(lex) == JSON_TOKEN_COMMA) @@ -1658,6 +1738,8 @@ json_lex(JsonLexContext *lex) if (lex == &failed_oom || lex->inc_state == &failed_inc_oom) return JSON_OUT_OF_MEMORY; + lex->token_is_identifier = false; + if (lex->incremental) { if (lex->inc_state->partial_completed) @@ -1820,13 +1902,14 @@ json_lex(JsonLexContext *lex) /* * Add any remaining alphanumeric chars. This takes care of the * {null, false, true} literals as well as any trailing - * alphanumeric junk on non-string tokens. + * alphanumeric junk on non-string tokens. json5 identifiers can + * also contain '$'. */ for (size_t i = added; i < lex->input_length; i++) { char cc = lex->input[i]; - if (JSON_ALPHANUMERIC_CHAR(cc)) + if (JSON_ALPHANUMERIC_CHAR(cc) || (lex->json5 && cc == '$')) { jsonapi_appendStringInfoCharMacro(ptok, cc); added++; @@ -1884,6 +1967,7 @@ json_lex(JsonLexContext *lex) * json_lex. */ lex->token_type = dummy_lex.token_type; + lex->token_is_identifier = dummy_lex.token_is_identifier; lex->line_number = dummy_lex.line_number; /* @@ -2141,7 +2225,8 @@ json_lex(JsonLexContext *lex) * the whole word as an unexpected token, rather than just * some unintuitive prefix thereof. */ - for (p = s; p < end && JSON_ALPHANUMERIC_CHAR(*p); p++) + for (p = s; p < end && (JSON_ALPHANUMERIC_CHAR(*p) || + (lex->json5 && *p == '$')); p++) /* skip */ ; /* @@ -2176,11 +2261,33 @@ json_lex(JsonLexContext *lex) lex->token_type = JSON_TOKEN_TRUE; else if (memcmp(s, "null", 4) == 0) lex->token_type = JSON_TOKEN_NULL; + else if (lex->json5) + goto json5_identifier; else return JSON_INVALID_TOKEN; } else if (p - s == 5 && memcmp(s, "false", 5) == 0) lex->token_type = JSON_TOKEN_FALSE; + else if (lex->json5) + { + /* Infinity/NaN handled with number extensions */ +json5_identifier: + if ((*s >= 'a' && *s <= 'z') || + (*s >= 'A' && *s <= 'Z') || + *s == '_' || *s == '$' || + IS_HIGHBIT_SET(*s)) + { + lex->token_type = JSON_TOKEN_STRING; + lex->token_is_identifier = true; + if (lex->need_escapes) + { + jsonapi_resetStringInfo(lex->strval); + jsonapi_appendBinaryStringInfo(lex->strval, s, p - s); + } + } + else + return JSON_INVALID_TOKEN; + } else return JSON_INVALID_TOKEN; } @@ -2560,9 +2667,13 @@ json_lex_number(JsonLexContext *lex, const char *s, /* * Check for trailing garbage. As in json_lex(), any alphanumeric stuff * here should be considered part of the token for error-reporting - * purposes. + * purposes; in json5 that includes '$'. The incremental partial-token + * code accumulates the same characters, so the two must agree on where + * the token ends. */ - for (; len < lex->input_length && JSON_ALPHANUMERIC_CHAR(*s); s++, len++) + for (; len < lex->input_length && + (JSON_ALPHANUMERIC_CHAR(*s) || (lex->json5 && *s == '$')); + s++, len++) error = true; if (total_len != NULL) diff --git a/src/include/common/jsonapi.h b/src/include/common/jsonapi.h index 8b4d8d818c7..d19f03f8556 100644 --- a/src/include/common/jsonapi.h +++ b/src/include/common/jsonapi.h @@ -109,6 +109,8 @@ typedef struct JsonLexContext bool incremental; bool json5; JsonTokenType token_type; + /* current token is a json5 unquoted identifier, only legal as a key */ + bool token_is_identifier; int lex_level; uint32 flags; int line_number; /* line number, starting from 1 */ diff --git a/src/test/modules/test_json_parser/json5_keys.json5 b/src/test/modules/test_json_parser/json5_keys.json5 new file mode 100644 index 00000000000..20d710bc65b --- /dev/null +++ b/src/test/modules/test_json_parser/json5_keys.json5 @@ -0,0 +1,12 @@ +{ + a: 1, + _b: 2, + $c: 3, + d1: 4, + true: 5, + false: 6, + null: 7, + "quoted": 8, + nested: { x: 1, y: { z: 2 } }, + arr: [ { k: 1 }, { k: 2 } ] +} diff --git a/src/test/modules/test_json_parser/json5_keys.out b/src/test/modules/test_json_parser/json5_keys.out new file mode 100644 index 00000000000..55647680f6b --- /dev/null +++ b/src/test/modules/test_json_parser/json5_keys.out @@ -0,0 +1,29 @@ +{ +"a": 1, +"_b": 2, +"$c": 3, +"d1": 4, +"true": 5, +"false": 6, +"null": 7, +"quoted": 8, +"nested": { +"x": 1, +"y": { +"z": 2 +} + +} +, +"arr": [ +{ +"k": 1 +} +, +{ +"k": 2 +} + +] + +} diff --git a/src/test/modules/test_json_parser/t/005_test_json5.pl b/src/test/modules/test_json_parser/t/005_test_json5.pl index 7e2519cd427..75d2e7b2ae8 100644 --- a/src/test/modules/test_json_parser/t/005_test_json5.pl +++ b/src/test/modules/test_json_parser/t/005_test_json5.pl @@ -31,11 +31,18 @@ my @features = ( file => 'json5_comments', error => qr/Token "\/" is invalid/, }, - { name => 'trailing commas', file => 'json5_trailing_commas' },); + { name => 'trailing commas', file => 'json5_trailing_commas' }, + { name => 'unquoted keys', file => 'json5_keys' },); # Inputs that stay invalid even in json5 mode. my @json5_invalid = ( - [ 'doubled trailing comma', '[1,,]' ],); + [ 'doubled trailing comma', '[1,,]' ], + [ 'digit-led key', '{ 1a: 1 }' ], + [ 'identifier value in object', '{ a: b }' ], + [ 'identifier value in array', '[a]' ], + [ 'bare identifier', 'undefined' ], + [ 'dollar after number', '2$' ], + [ 'dollar after number in array', '[25$]' ],); # Write $content to a temp file and return the file name. sub inline_file -- 2.43.0