From 0543a41387526f6bfa8e883d8df89c8e83d92d33 Mon Sep 17 00:00:00 2001 From: Andrew Dunstan Date: Thu, 9 Jul 2026 07:40:47 -0400 Subject: [PATCH] Support JSON5 syntax in the recursive descent JSON parser Add setJsonLexContextJSON5(), which enables a set of JSON5 syntax extensions on a JsonLexContext: comments (both "//" and slash-star forms), trailing commas in arrays and objects, single-quoted strings, and unquoted (identifier) object keys. The feature is only supported by the recursive descent parser invoked via pg_parse_json(); enabling it on an incremental lexing context returns JSON_INVALID_LEXER_TYPE, the same error pg_parse_json() itself returns when called with an incremental lexing context, since the table-driven incremental grammar does not know about the JSON5 extensions to the grammar. A new JSON_TOKEN_IDENTIFIER token type represents a bare identifier, which is only ever produced in JSON5 mode. It is accepted by the grammar wherever a string key is expected, but parse_scalar() still rejects it as a value, so unquoted identifiers remain illegal outside of key position, matching the JSON5 grammar. Add test_json_parser_json5, a standalone test program exercising the recursive descent parser in JSON5 mode, along with a TAP test covering the new syntax and confirming it is still rejected when JSON5 mode is off. --- src/common/jsonapi.c | 209 ++++++++++-- src/include/common/jsonapi.h | 27 ++ src/test/modules/test_json_parser/.gitignore | 1 + src/test/modules/test_json_parser/Makefile | 9 +- src/test/modules/test_json_parser/README | 8 +- src/test/modules/test_json_parser/meson.build | 23 +- .../test_json_parser/t/005_test_json5.pl | 147 +++++++++ .../test_json_parser/test_json_parser_json5.c | 298 ++++++++++++++++++ 8 files changed, 692 insertions(+), 30 deletions(-) create mode 100644 src/test/modules/test_json_parser/t/005_test_json5.pl create mode 100644 src/test/modules/test_json_parser/test_json_parser_json5.c diff --git a/src/common/jsonapi.c b/src/common/jsonapi.c index d3860197dad..161c4b4556f 100644 --- a/src/common/jsonapi.c +++ b/src/common/jsonapi.c @@ -322,6 +322,18 @@ lex_expect(JsonParseContext ctx, JsonLexContext *lex, JsonTokenType token) return report_parse_error(ctx, lex); } +/* + * lex_is_json5 + * + * is JSON5 syntax enabled for this lexing context? See + * setJsonLexContextJSON5(). + */ +static inline bool +lex_is_json5(JsonLexContext *lex) +{ + return (lex->flags & JSONLEX_JSON5) != 0; +} + /* chars to consider as part of an alphanumeric token */ #define JSON_ALPHANUMERIC_CHAR(c) \ (((c) >= 'a' && (c) <= 'z') || \ @@ -558,6 +570,30 @@ setJsonLexContextOwnsTokens(JsonLexContext *lex, bool owned_by_context) lex->flags &= ~JSONLEX_CTX_OWNS_TOKENS; } +/* + * setJsonLexContextJSON5 + * + * See the declaration of this function in jsonapi.h for the details of what + * enabling JSON5 mode does. It is only supported by the recursive descent + * parser invoked via pg_parse_json(), so it may not be enabled on an + * incremental lexing context; JSON_INVALID_LEXER_TYPE is returned in that + * case, matching the error pg_parse_json() itself returns when called with + * an incremental lexing context. + */ +JsonParseErrorType +setJsonLexContextJSON5(JsonLexContext *lex, bool enable) +{ + if (lex->incremental) + return JSON_INVALID_LEXER_TYPE; + + if (enable) + lex->flags |= JSONLEX_JSON5; + else + lex->flags &= ~JSONLEX_JSON5; + + return JSON_SUCCESS; +} + static inline bool inc_lex_level(JsonLexContext *lex) { @@ -837,6 +873,10 @@ json_count_array_elements(JsonLexContext *lex, int *elements) result = json_lex(©lex); if (result != JSON_SUCCESS) return result; + + /* JSON5 permits a trailing comma before the closing bracket */ + if (lex_is_json5(©lex) && lex_peek(©lex) == JSON_TOKEN_ARRAY_END) + break; } } result = lex_expect(JSON_PARSE_ARRAY_NEXT, ©lex, @@ -1325,7 +1365,8 @@ parse_object_field(JsonLexContext *lex, const JsonSemAction *sem) JsonTokenType tok; JsonParseErrorType result; - if (lex_peek(lex) != JSON_TOKEN_STRING) + if (lex_peek(lex) != JSON_TOKEN_STRING && + !(lex_is_json5(lex) && lex_peek(lex) == JSON_TOKEN_IDENTIFIER)) return report_parse_error(JSON_PARSE_STRING, lex); if ((ostart != NULL || oend != NULL) && lex->need_escapes) { @@ -1430,12 +1471,18 @@ parse_object(JsonLexContext *lex, const JsonSemAction *sem) switch (tok) { case JSON_TOKEN_STRING: + case JSON_TOKEN_IDENTIFIER: /* only reachable in JSON5 mode */ result = parse_object_field(lex, sem); while (result == JSON_SUCCESS && lex_peek(lex) == JSON_TOKEN_COMMA) { result = json_lex(lex); if (result != JSON_SUCCESS) break; + + /* JSON5 permits a trailing comma before the closing brace */ + if (lex_is_json5(lex) && lex_peek(lex) == JSON_TOKEN_OBJECT_END) + break; + result = parse_object_field(lex, sem); } break; @@ -1548,6 +1595,11 @@ parse_array(JsonLexContext *lex, const JsonSemAction *sem) result = json_lex(lex); if (result != JSON_SUCCESS) break; + + /* JSON5 permits a trailing comma before the closing bracket */ + if (lex_is_json5(lex) && lex_peek(lex) == JSON_TOKEN_ARRAY_END) + break; + result = parse_array_element(lex, sem); } } @@ -1849,13 +1901,62 @@ json_lex(JsonLexContext *lex) /* end of partial token processing */ } - /* Skip leading whitespace. */ - while (s < end && (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r')) + /* + * Skip leading whitespace, along with JSON5 comments, if enabled. JSON5 + * allows comments anywhere whitespace is allowed, so keep alternating + * between the two until neither one advances any further. + */ + for (;;) { - if (*s++ == '\n') + while (s < end && (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r')) { - ++lex->line_number; - lex->line_start = s; + if (*s++ == '\n') + { + ++lex->line_number; + lex->line_start = s; + } + } + + if (!lex_is_json5(lex) || s >= end || *s != '/' || s + 1 >= end) + break; + + if (s[1] == '/') + { + /* "//" comment: skip through the end of the line. */ + s += 2; + while (s < end && *s != '\n') + s++; + } + else if (s[1] == '*') + { + /* block comment: skip through the closing delimiter */ + const char *comment_start = s; + + s += 2; + while (s < end && !(s[0] == '*' && s + 1 < end && s[1] == '/')) + { + if (*s == '\n') + { + ++lex->line_number; + lex->line_start = s + 1; + } + s++; + } + + if (s >= end) + { + lex->token_start = comment_start; + lex->prev_token_terminator = lex->token_terminator; + lex->token_terminator = end; + return JSON_UNTERMINATED_COMMENT; + } + + s += 2; /* skip over the closing delimiter */ + } + else + { + /* lone '/': not a comment, let the normal lexer deal with it */ + break; } } lex->token_start = s; @@ -1910,6 +2011,20 @@ json_lex(JsonLexContext *lex) return result; lex->token_type = JSON_TOKEN_STRING; break; + case '\'': + if (!lex_is_json5(lex)) + { + /* not a legal token character outside JSON5 mode */ + lex->prev_token_terminator = lex->token_terminator; + lex->token_terminator = s + 1; + return JSON_INVALID_TOKEN; + } + /* JSON5 also allows single-quoted strings. */ + result = json_lex_string(lex); + if (result != JSON_SUCCESS) + return result; + lex->token_type = JSON_TOKEN_STRING; + break; case '-': /* Negative number. */ result = json_lex_number(lex, s + 1, NULL, NULL); @@ -1936,12 +2051,14 @@ json_lex(JsonLexContext *lex) default: { const char *p; + bool is_keyword; /* * We're not dealing with a string, number, legal * punctuation mark, or end of string. The only legal - * tokens we might find here are true, false, and null, - * but for error reporting purposes we scan until we see a + * tokens we might find here are true, false, and null + * (plus, in JSON5 mode, a bare identifier), but for error + * reporting purposes we scan until we see a * non-alphanumeric character. That way, we can report * the whole word as an unexpected token, rather than just * some unintuitive prefix thereof. @@ -1969,25 +2086,48 @@ json_lex(JsonLexContext *lex) } /* - * We've got a real alphanumeric token here. If it - * happens to be true, false, or null, all is well. If - * not, error out. + * We've got a real alphanumeric token here. Check + * whether it happens to be true, false, or null. */ lex->prev_token_terminator = lex->token_terminator; lex->token_terminator = p; - if (p - s == 4) - { - if (memcmp(s, "true", 4) == 0) - lex->token_type = JSON_TOKEN_TRUE; - else if (memcmp(s, "null", 4) == 0) - lex->token_type = JSON_TOKEN_NULL; - else - return JSON_INVALID_TOKEN; - } + is_keyword = true; + if (p - s == 4 && memcmp(s, "true", 4) == 0) + lex->token_type = JSON_TOKEN_TRUE; + else if (p - s == 4 && memcmp(s, "null", 4) == 0) + lex->token_type = JSON_TOKEN_NULL; else if (p - s == 5 && memcmp(s, "false", 5) == 0) lex->token_type = JSON_TOKEN_FALSE; else - return JSON_INVALID_TOKEN; + is_keyword = false; + + if (!is_keyword) + { + /* + * Not a keyword. In JSON5 mode, treat it as a bare + * identifier (e.g. an unquoted object key); the + * grammar is responsible for rejecting it wherever an + * identifier isn't allowed. Otherwise it's an error. + */ + if (!lex_is_json5(lex)) + return JSON_INVALID_TOKEN; + + lex->token_type = JSON_TOKEN_IDENTIFIER; + if (lex->need_escapes) + { +#ifdef JSONAPI_USE_PQEXPBUFFER + /* make sure initialization succeeded */ + if (lex->strval == NULL) + return JSON_OUT_OF_MEMORY; +#endif + jsonapi_resetStringInfo(lex->strval); + jsonapi_appendBinaryStringInfo(lex->strval, s, p - s); +#ifdef JSONAPI_USE_PQEXPBUFFER + if (PQExpBufferBroken(lex->strval)) + return JSON_OUT_OF_MEMORY; +#endif + } + } } } /* end of switch */ } @@ -2015,6 +2155,9 @@ json_lex_string(JsonLexContext *lex) const char *s; const char *const end = lex->input + lex->input_length; int hi_surrogate = -1; + const char quote = *lex->token_start; /* '"', or in JSON5 mode '\'' */ + + Assert(quote == '"' || (lex_is_json5(lex) && quote == '\'')); /* Convenience macros for error exits */ #define FAIL_OR_INCOMPLETE_AT_CHAR_START(code) \ @@ -2057,7 +2200,7 @@ json_lex_string(JsonLexContext *lex) /* Premature end of the string. */ if (s >= end) FAIL_OR_INCOMPLETE_AT_CHAR_START(JSON_INVALID_TOKEN); - else if (*s == '"') + else if (*s == quote) break; else if (*s == '\\') { @@ -2165,6 +2308,19 @@ json_lex_string(JsonLexContext *lex) case '/': jsonapi_appendStringInfoChar(lex->strval, *s); break; + case '\'': + if (!lex_is_json5(lex)) + { + /* + * Not a valid string escape, so signal error. We + * adjust token_start so that just the escape + * sequence is reported, not the whole string. + */ + lex->token_start = s; + FAIL_AT_CHAR_END(JSON_ESCAPING_INVALID); + } + jsonapi_appendStringInfoChar(lex->strval, *s); + break; case 'b': jsonapi_appendStringInfoChar(lex->strval, '\b'); break; @@ -2191,7 +2347,8 @@ json_lex_string(JsonLexContext *lex) FAIL_AT_CHAR_END(JSON_ESCAPING_INVALID); } } - else if (strchr("\"\\/bfnrt", *s) == NULL) + else if (strchr("\"\\/bfnrt", *s) == NULL && + !(lex_is_json5(lex) && *s == '\'')) { /* * Simpler processing if we're not bothered about de-escaping @@ -2217,13 +2374,13 @@ json_lex_string(JsonLexContext *lex) */ while (p < end - sizeof(Vector8) && !pg_lfind8('\\', (const uint8 *) p, sizeof(Vector8)) && - !pg_lfind8('"', (const uint8 *) p, sizeof(Vector8)) && + !pg_lfind8((uint8) quote, (const uint8 *) p, sizeof(Vector8)) && !pg_lfind8_le(31, (const uint8 *) p, sizeof(Vector8))) p += sizeof(Vector8); for (; p < end; p++) { - if (*p == '\\' || *p == '"') + if (*p == '\\' || *p == quote) break; else if ((unsigned char) *p <= 31) { @@ -2523,6 +2680,8 @@ json_errdetail(JsonParseErrorType error, JsonLexContext *lex) case JSON_INVALID_TOKEN: json_token_error(lex, "Token \"%.*s\" is invalid."); break; + case JSON_UNTERMINATED_COMMENT: + return _("Unterminated \"/*\" comment."); case JSON_OUT_OF_MEMORY: /* should have been handled above; use the error path */ break; diff --git a/src/include/common/jsonapi.h b/src/include/common/jsonapi.h index 85cc9a11d97..395edb4e9d8 100644 --- a/src/include/common/jsonapi.h +++ b/src/include/common/jsonapi.h @@ -29,6 +29,13 @@ typedef enum JsonTokenType JSON_TOKEN_FALSE, JSON_TOKEN_NULL, JSON_TOKEN_END, + + /* + * Only produced when JSON5 mode is enabled (see + * setJsonLexContextJSON5()), for a bare identifier such as an unquoted + * object key. + */ + JSON_TOKEN_IDENTIFIER, } JsonTokenType; typedef enum JsonParseErrorType @@ -49,6 +56,7 @@ typedef enum JsonParseErrorType JSON_EXPECTED_OBJECT_NEXT, JSON_EXPECTED_STRING, JSON_INVALID_TOKEN, + JSON_UNTERMINATED_COMMENT, JSON_OUT_OF_MEMORY, JSON_UNICODE_CODE_POINT_ZERO, JSON_UNICODE_ESCAPE_FORMAT, @@ -93,10 +101,12 @@ typedef struct JsonIncrementalState JsonIncrementalState; * * JSONLEX_FREE_STRUCT/STRVAL are used to drive freeJsonLexContext. * JSONLEX_CTX_OWNS_TOKENS is used by setJsonLexContextOwnsTokens. + * JSONLEX_JSON5 is used by setJsonLexContextJSON5. */ #define JSONLEX_FREE_STRUCT (1 << 0) #define JSONLEX_FREE_STRVAL (1 << 1) #define JSONLEX_CTX_OWNS_TOKENS (1 << 2) +#define JSONLEX_JSON5 (1 << 3) typedef struct JsonLexContext { const char *input; @@ -249,6 +259,23 @@ extern JsonLexContext *makeJsonLexContextIncremental(JsonLexContext *lex, extern void setJsonLexContextOwnsTokens(JsonLexContext *lex, bool owned_by_context); +/* + * Enables or disables JSON5 syntax extensions for the given lexing context: + * comments (both "// ..." and slash-star ... star-slash forms), trailing + * commas in arrays and objects, single-quoted strings, and unquoted + * (identifier) object keys. + * + * JSON5 mode is only supported for the recursive descent parser invoked via + * pg_parse_json(); enabling it on a context set up for incremental parsing + * (see makeJsonLexContextIncremental()) returns JSON_INVALID_LEXER_TYPE + * without making any change, the same error pg_parse_json() itself returns + * when called with an incremental lexing context. + * + * By default, JSON5 mode is disabled. + */ +extern JsonParseErrorType setJsonLexContextJSON5(JsonLexContext *lex, + bool enable); + extern void freeJsonLexContext(JsonLexContext *lex); /* lex one token */ diff --git a/src/test/modules/test_json_parser/.gitignore b/src/test/modules/test_json_parser/.gitignore index f032d1e4f90..b2d8d8bc0ae 100644 --- a/src/test/modules/test_json_parser/.gitignore +++ b/src/test/modules/test_json_parser/.gitignore @@ -2,3 +2,4 @@ tmp_check test_json_parser_perf test_json_parser_incremental test_json_parser_incremental_shlib +test_json_parser_json5 diff --git a/src/test/modules/test_json_parser/Makefile b/src/test/modules/test_json_parser/Makefile index af3f19424ed..9afd2b408ad 100644 --- a/src/test/modules/test_json_parser/Makefile +++ b/src/test/modules/test_json_parser/Makefile @@ -4,9 +4,9 @@ PGAPPICON = win32 TAP_TESTS = 1 -OBJS = test_json_parser_incremental.o test_json_parser_perf.o $(WIN32RES) +OBJS = test_json_parser_incremental.o test_json_parser_perf.o test_json_parser_json5.o $(WIN32RES) -EXTRA_CLEAN = test_json_parser_incremental$(X) test_json_parser_incremental_shlib$(X) test_json_parser_perf$(X) +EXTRA_CLEAN = test_json_parser_incremental$(X) test_json_parser_incremental_shlib$(X) test_json_parser_perf$(X) test_json_parser_json5$(X) ifdef USE_PGXS PG_CONFIG = pg_config @@ -19,7 +19,7 @@ include $(top_builddir)/src/Makefile.global include $(top_srcdir)/contrib/contrib-global.mk endif -all: test_json_parser_incremental$(X) test_json_parser_incremental_shlib$(X) test_json_parser_perf$(X) +all: test_json_parser_incremental$(X) test_json_parser_incremental_shlib$(X) test_json_parser_perf$(X) test_json_parser_json5$(X) %.o: $(top_srcdir)/$(subdir)/%.c @@ -32,6 +32,9 @@ test_json_parser_incremental_shlib$(X): test_json_parser_incremental.o $(WIN32RE test_json_parser_perf$(X): test_json_parser_perf.o $(WIN32RES) $(CC) $(CFLAGS) $^ $(PG_LIBS_INTERNAL) $(LDFLAGS) $(LDFLAGS_EX) $(PG_LIBS) $(LIBS) -o $@ +test_json_parser_json5$(X): test_json_parser_json5.o $(WIN32RES) + $(CC) $(CFLAGS) $^ $(PG_LIBS_INTERNAL) $(LDFLAGS) $(LDFLAGS_EX) $(PG_LIBS) $(LIBS) -o $@ + speed-check: test_json_parser_perf$(X) @echo Standard parser: time ./test_json_parser_perf 10000 $(top_srcdir)/$(subdir)/tiny.json diff --git a/src/test/modules/test_json_parser/README b/src/test/modules/test_json_parser/README index 61e7c78d588..b68f5e241b9 100644 --- a/src/test/modules/test_json_parser/README +++ b/src/test/modules/test_json_parser/README @@ -1,7 +1,7 @@ Module `test_json_parser` ========================= -This module contains two programs for testing the json parsers. +This module contains three programs for testing the json parsers. - `test_json_parser_incremental` is for testing the incremental parser, It reads in a file and passes it in very small chunks (default is 60 bytes at a @@ -20,6 +20,12 @@ This module contains two programs for testing the json parsers. using the non-recursive parser, the input is passed to the parser in a single chunk. The results are thus comparable to those of the standard parser. +- `test_json_parser_json5` is for testing JSON5 mode of the standard + recursive descent parser (see `setJsonLexContextJSON5()`). It parses its + input in a single chunk, with JSON5 mode enabled unless "-p" (plain JSON) + is given. As with `test_json_parser_incremental`, "-s" specifies using + semantic routines to re-output the json. The required non-option argument + is the input file name. The sample input file is a small, sanitized extract from a list of `delicious` bookmarks taken some years ago, all wrapped in a single json diff --git a/src/test/modules/test_json_parser/meson.build b/src/test/modules/test_json_parser/meson.build index 2688686e37b..6b2c60e11c7 100644 --- a/src/test/modules/test_json_parser/meson.build +++ b/src/test/modules/test_json_parser/meson.build @@ -50,6 +50,25 @@ test_json_parser_perf = executable('test_json_parser_perf', }, ) +test_json_parser_json5_sources = files( + 'test_json_parser_json5.c', +) + +if host_system == 'windows' + test_json_parser_json5_sources += rc_bin_gen.process(win32ver_rc, extra_args: [ + '--NAME', 'test_json_parser_json5', + '--FILEDESC', 'standalone json parser tester', + ]) +endif + +test_json_parser_json5 = executable('test_json_parser_json5', + test_json_parser_json5_sources, + dependencies: [frontend_code], + kwargs: default_bin_args + { + 'install': false, + }, +) + tests += { 'name': 'test_json_parser', 'sd': meson.current_source_dir(), @@ -59,12 +78,14 @@ tests += { 't/001_test_json_parser_incremental.pl', 't/002_inline.pl', 't/003_test_semantic.pl', - 't/004_test_parser_perf.pl' + 't/004_test_parser_perf.pl', + 't/005_test_json5.pl', ], 'deps': [ test_json_parser_incremental, test_json_parser_incremental_shlib, test_json_parser_perf, + test_json_parser_json5, ], }, } diff --git a/src/test/modules/test_json_parser/t/005_test_json5.pl b/src/test/modules/test_json_parser/t/005_test_json5.pl new file mode 100644 index 00000000000..dffd2443fea --- /dev/null +++ b/src/test/modules/test_json_parser/t/005_test_json5.pl @@ -0,0 +1,147 @@ + +# Copyright (c) 2024-2026, PostgreSQL Global Development Group + +# Test JSON5 support in the recursive descent JSON parser: comments, +# trailing commas, single-quoted strings and unquoted object keys. + +use strict; +use warnings FATAL => 'all'; + +use PostgreSQL::Test::Utils; +use Test::More; + +use File::Temp qw(tempfile); + +my $dir = PostgreSQL::Test::Utils::tempdir; + +sub run_json5 +{ + my ($json, @extra_args) = @_; + + my ($fh, $fname) = tempfile(DIR => $dir); + print $fh $json; + close($fh); + + return run_command( + [ "test_json_parser_json5", @extra_args, $fname ]); +} + +# Test that $json succeeds in JSON5 mode, but fails outside of it (unless +# plain_ok is set, for cases which are also valid plain JSON). +sub test_json5_only +{ + local $Test::Builder::Level = $Test::Builder::Level + 1; + + my ($name, $json, %params) = @_; + + my ($stdout, $stderr) = run_json5($json); + is($stdout, "SUCCESS!", "$name: succeeds in JSON5 mode"); + is($stderr, "", "$name: no error output in JSON5 mode"); + + ($stdout, $stderr) = run_json5($json, "-p"); + if ($params{plain_ok}) + { + is($stdout, "SUCCESS!", "$name: also succeeds as plain JSON"); + is($stderr, "", "$name: no error output as plain JSON"); + } + else + { + unlike($stdout, qr/SUCCESS/, "$name: fails as plain JSON"); + isnt($stderr, "", "$name: error output as plain JSON"); + } +} + +sub test_json5_error +{ + local $Test::Builder::Level = $Test::Builder::Level + 1; + + my ($name, $json, $error) = @_; + + my ($stdout, $stderr) = run_json5($json); + unlike($stdout, qr/SUCCESS/, "$name: fails in JSON5 mode"); + like($stderr, $error, "$name: correct error output"); +} + +# Test that the semantic (-s) output for $json, parsed in JSON5 mode, +# matches $expected (which should be valid, canonical JSON). +sub test_json5_semantic +{ + local $Test::Builder::Level = $Test::Builder::Level + 1; + + my ($name, $json, $expected) = @_; + + my ($stdout, $stderr) = run_json5($json, "-s"); + is($stderr, "", "$name: no error output"); + + # normalize whitespace the same way for both sides + (my $got = $stdout) =~ s/\s+//g; + (my $want = $expected) =~ s/\s+//g; + is($got, $want, "$name: correct semantic output"); +} + +# Comments +test_json5_only("line comment", "// hello\n123"); +test_json5_only("line comment at eol", "123 // hello"); +test_json5_only("block comment", "/* hello */ 123"); +test_json5_only("block comment inside array", + "[1, /* comment */ 2]"); +test_json5_only( + "multi-line block comment", + "[1,\n/* a\nmulti\nline\ncomment */\n2]"); + +test_json5_error("unterminated block comment", + "/* hello", qr/Unterminated.*comment/); + +# Trailing commas +test_json5_only("trailing comma in array", "[1, 2, 3,]"); +test_json5_only("trailing comma in object", '{"a": 1, "b": 2,}'); +test_json5_only("trailing comma in nested array", "[[1,],[2,],]"); + +# a lone trailing comma is not enough to make an otherwise-empty +# array or object valid +test_json5_error("array with only a comma", "[,]", + qr/Expected JSON value, but found ","/); +test_json5_error("object with only a comma", "{,}", + qr/Expected string or "}", but found ","/); + +# Single-quoted strings +test_json5_only("single-quoted string", "'hello'"); +test_json5_only("single-quoted string in array", "['hello', 'world']"); +test_json5_only( + "single-quoted string with escaped single quote", "'it\\'s'"); +test_json5_only( + "double-quoted string with escaped single quote inside", '"it\\\'s"'); + +# Unquoted object keys +test_json5_only("unquoted key", "{foo: 1}"); +test_json5_only("unquoted key with underscore", "{_foo_bar: 1}"); +test_json5_only( + "mixed quoted and unquoted keys", + '{foo: 1, "bar": 2, baz: 3}'); + +# identifiers are only legal in key position, not as bare values, even in +# JSON5 mode +test_json5_error("bare identifier as value", "[foo]", + qr/Expected JSON value, but found "foo"/); +test_json5_error("bare identifier as top-level value", "foo", + qr/Expected JSON value, but found "foo"/); + +# semantic (structural) checks +test_json5_semantic("unquoted key round-trip", "{foo: 'bar'}", + '{"foo": "bar"}'); +test_json5_semantic( + "trailing comma round-trip", + "[1, 2, 3,]", + "[1,\n2,\n3]"); +test_json5_semantic( + "comments and trailing commas combined", + q{ + { + // this is a comment + foo: 'bar', /* another comment */ + baz: [1, 2, 3,], + } + }, + '{"foo": "bar", "baz": [1, 2, 3]}'); + +done_testing(); diff --git a/src/test/modules/test_json_parser/test_json_parser_json5.c b/src/test/modules/test_json_parser/test_json_parser_json5.c new file mode 100644 index 00000000000..857b0796500 --- /dev/null +++ b/src/test/modules/test_json_parser/test_json_parser_json5.c @@ -0,0 +1,298 @@ +/*------------------------------------------------------------------------- + * + * test_json_parser_json5.c + * Test program for JSON5 mode of the recursive descent JSON parser + * + * Copyright (c) 2024-2026, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/test/modules/test_json_parser/test_json_parser_json5.c + * + * This program parses its input with the recursive descent (non-incremental) + * JSON parser, pg_parse_json(). Unless "-p" (plain) is given, JSON5 mode is + * enabled, so that comments, trailing commas, single-quoted strings and + * unquoted object keys are all accepted in addition to standard JSON. + * + * If the -s flag is given, the program does semantic processing, mirroring + * back the input as standard JSON (albeit with white space changes). This + * can be used to confirm that JSON5-only syntax is interpreted correctly, + * not just that it's accepted. + * + * The argument specifies the file containing the JSON input. + * + *------------------------------------------------------------------------- + */ + +#include "postgres_fe.h" + +#include + +#include "common/jsonapi.h" +#include "common/logging.h" +#include "lib/stringinfo.h" +#include "mb/pg_wchar.h" +#include "pg_getopt.h" + +#define BUFSIZE 6000 + +typedef struct DoState +{ + bool elem_is_first; + StringInfo buf; +} DoState; + +static void usage(const char *progname); +static void escape_json(StringInfo buf, const char *str); + +/* semantic action functions for parser */ +static JsonParseErrorType do_object_start(void *state); +static JsonParseErrorType do_object_end(void *state); +static JsonParseErrorType do_object_field_start(void *state, char *fname, bool isnull); +static JsonParseErrorType do_array_start(void *state); +static JsonParseErrorType do_array_end(void *state); +static JsonParseErrorType do_array_element_start(void *state, bool isnull); +static JsonParseErrorType do_scalar(void *state, char *token, JsonTokenType tokentype); + +static JsonSemAction sem = { + .object_start = do_object_start, + .object_end = do_object_end, + .object_field_start = do_object_field_start, + .array_start = do_array_start, + .array_end = do_array_end, + .array_element_start = do_array_element_start, + .scalar = do_scalar +}; + +int +main(int argc, char **argv) +{ + char buff[BUFSIZE]; + FILE *json_file; + JsonParseErrorType result; + JsonLexContext *lex; + StringInfoData json; + int n_read; + bool json5 = true; + bool need_strings = false; + const JsonSemAction *testsem = &nullSemAction; + char *testfile; + int c; + DoState state; + + pg_logging_init(argv[0]); + + while ((c = getopt(argc, argv, "ps")) != -1) + { + switch (c) + { + case 'p': /* plain JSON, not JSON5 */ + json5 = false; + break; + case 's': /* do semantic processing */ + testsem = &sem; + need_strings = true; + break; + } + } + + if (optind < argc) + { + testfile = argv[optind]; + optind++; + } + else + { + usage(argv[0]); + exit(1); + } + + initStringInfo(&json); + + if ((json_file = fopen(testfile, PG_BINARY_R)) == NULL) + pg_fatal("error opening input: %m"); + + while ((n_read = fread(buff, 1, BUFSIZE, json_file)) > 0) + appendBinaryStringInfo(&json, buff, n_read); + fclose(json_file); + + lex = makeJsonLexContextCstringLen(NULL, json.data, json.len, + PG_UTF8, need_strings); + if (json5 && setJsonLexContextJSON5(lex, true) != JSON_SUCCESS) + pg_fatal("could not enable JSON5 mode"); + + if (testsem == &sem) + { + state.elem_is_first = true; + state.buf = makeStringInfo(); + sem.semstate = &state; + } + + result = pg_parse_json(lex, testsem); + + if (result != JSON_SUCCESS) + { + fprintf(stderr, "%s\n", json_errdetail(result, lex)); + freeJsonLexContext(lex); + free(json.data); + return 1; + } + + if (!need_strings) + printf("SUCCESS!\n"); + + freeJsonLexContext(lex); + free(json.data); + + return 0; +} + +/* + * The semantic routines here essentially just output the same json, except + * for white space and, in JSON5 mode, resolving any JSON5-only syntax into + * plain JSON. The result should be able to be fed to any JSON processor + * such as jq for validation. + */ + +static JsonParseErrorType +do_object_start(void *state) +{ + DoState *_state = (DoState *) state; + + printf("{\n"); + _state->elem_is_first = true; + + return JSON_SUCCESS; +} + +static JsonParseErrorType +do_object_end(void *state) +{ + DoState *_state = (DoState *) state; + + printf("\n}\n"); + _state->elem_is_first = false; + + return JSON_SUCCESS; +} + +static JsonParseErrorType +do_object_field_start(void *state, char *fname, bool isnull) +{ + DoState *_state = (DoState *) state; + + if (!_state->elem_is_first) + printf(",\n"); + resetStringInfo(_state->buf); + escape_json(_state->buf, fname); + printf("%s: ", _state->buf->data); + _state->elem_is_first = false; + free(fname); + + return JSON_SUCCESS; +} + +static JsonParseErrorType +do_array_start(void *state) +{ + DoState *_state = (DoState *) state; + + printf("[\n"); + _state->elem_is_first = true; + + return JSON_SUCCESS; +} + +static JsonParseErrorType +do_array_end(void *state) +{ + DoState *_state = (DoState *) state; + + printf("\n]\n"); + _state->elem_is_first = false; + + return JSON_SUCCESS; +} + +static JsonParseErrorType +do_array_element_start(void *state, bool isnull) +{ + DoState *_state = (DoState *) state; + + if (!_state->elem_is_first) + printf(",\n"); + _state->elem_is_first = false; + + return JSON_SUCCESS; +} + +static JsonParseErrorType +do_scalar(void *state, char *token, JsonTokenType tokentype) +{ + DoState *_state = (DoState *) state; + + if (tokentype == JSON_TOKEN_STRING) + { + resetStringInfo(_state->buf); + escape_json(_state->buf, token); + printf("%s", _state->buf->data); + } + else + printf("%s", token); + + free(token); + + return JSON_SUCCESS; +} + +/* copied from backend code */ +static void +escape_json(StringInfo buf, const char *str) +{ + const char *p; + + appendStringInfoCharMacro(buf, '"'); + for (p = str; *p; p++) + { + switch (*p) + { + case '\b': + appendStringInfoString(buf, "\\b"); + break; + case '\f': + appendStringInfoString(buf, "\\f"); + break; + case '\n': + appendStringInfoString(buf, "\\n"); + break; + case '\r': + appendStringInfoString(buf, "\\r"); + break; + case '\t': + appendStringInfoString(buf, "\\t"); + break; + case '"': + appendStringInfoString(buf, "\\\""); + break; + case '\\': + appendStringInfoString(buf, "\\\\"); + break; + default: + if ((unsigned char) *p < ' ') + appendStringInfo(buf, "\\u%04x", (int) *p); + else + appendStringInfoCharMacro(buf, *p); + break; + } + } + appendStringInfoCharMacro(buf, '"'); +} + +static void +usage(const char *progname) +{ + fprintf(stderr, + "Usage: %s [-p] [-s] filename\n" + " -p parse as plain JSON, not JSON5\n" + " -s do semantic processing\n", + progname); +} -- 2.43.0