From 65ff4ce5ef76f7ddebafcd15dc4410e1c290af7a Mon Sep 17 00:00:00 2001 From: Zsolt Parragi Date: Wed, 8 Jul 2026 23:06:23 +0000 Subject: [PATCH 6/8] Add JSON5 multi-line string support to the JSON lexer When json5 is set, accept backslash-newline line continuations inside strings, advancing line tracking, in normal and incremental modes. --- src/common/jsonapi.c | 60 +++++++++++++++---- .../test_json_parser/json5_multiline.json5 | 9 +++ .../test_json_parser/json5_multiline.out | 5 ++ .../test_json_parser/t/005_test_json5.pl | 38 ++++++++++++ 4 files changed, 101 insertions(+), 11 deletions(-) create mode 100644 src/test/modules/test_json_parser/json5_multiline.json5 create mode 100644 src/test/modules/test_json_parser/json5_multiline.out diff --git a/src/common/jsonapi.c b/src/common/jsonapi.c index 5920a3e1c7c..7085a102a3a 100644 --- a/src/common/jsonapi.c +++ b/src/common/jsonapi.c @@ -2657,6 +2657,28 @@ json_lex_string(JsonLexContext *lex) case 't': jsonapi_appendStringInfoChar(lex->strval, '\t'); break; + case '\n': + if (lex->json5) + { + ++lex->line_number; + lex->line_start = s + 1; + break; + } + lex->token_start = s; + FAIL_AT_CHAR_END(JSON_ESCAPING_INVALID); + break; + case '\r': + if (lex->json5) + { + if (s + 1 < end && *(s + 1) == '\n') + s++; + ++lex->line_number; + lex->line_start = s + 1; + break; + } + lex->token_start = s; + FAIL_AT_CHAR_END(JSON_ESCAPING_INVALID); + break; default: /* @@ -2668,18 +2690,34 @@ json_lex_string(JsonLexContext *lex) FAIL_AT_CHAR_END(JSON_ESCAPING_INVALID); } } - else if (strchr("\"\\/bfnrt", *s) == NULL && - !(lex->json5 && *s == '\'')) + else { - /* - * Simpler processing if we're not bothered about de-escaping - * - * It's very tempting to remove the strchr() call here and - * replace it with a switch statement, but testing so far has - * shown it's not a performance win. - */ - lex->token_start = s; - FAIL_AT_CHAR_END(JSON_ESCAPING_INVALID); + if (lex->json5 && *s == '\r') + { + if (s + 1 < end && *(s + 1) == '\n') + s++; + ++lex->line_number; + lex->line_start = s + 1; + } + else if (lex->json5 && *s == '\n') + { + ++lex->line_number; + lex->line_start = s + 1; + } + else if (strchr("\"\\/bfnrt", *s) == NULL && + !(lex->json5 && *s == '\'')) + { + /* + * Simpler processing if we're not bothered about + * de-escaping + * + * It's very tempting to remove the strchr() call here and + * replace it with a switch statement, but testing so far + * has shown it's not a performance win. + */ + lex->token_start = s; + FAIL_AT_CHAR_END(JSON_ESCAPING_INVALID); + } } } else diff --git a/src/test/modules/test_json_parser/json5_multiline.json5 b/src/test/modules/test_json_parser/json5_multiline.json5 new file mode 100644 index 00000000000..7a4050c5d75 --- /dev/null +++ b/src/test/modules/test_json_parser/json5_multiline.json5 @@ -0,0 +1,9 @@ +{ + a: 'line one \ +line two \ +line three', + b: "double one \ +double two", + c: 'crlf one \ +crlf two' +} diff --git a/src/test/modules/test_json_parser/json5_multiline.out b/src/test/modules/test_json_parser/json5_multiline.out new file mode 100644 index 00000000000..3a0ba5c2fe7 --- /dev/null +++ b/src/test/modules/test_json_parser/json5_multiline.out @@ -0,0 +1,5 @@ +{ +"a": "line one line two line three", +"b": "double one double two", +"c": "crlf one crlf two" +} diff --git a/src/test/modules/test_json_parser/t/005_test_json5.pl b/src/test/modules/test_json_parser/t/005_test_json5.pl index 62572f51c45..19755817e86 100644 --- a/src/test/modules/test_json_parser/t/005_test_json5.pl +++ b/src/test/modules/test_json_parser/t/005_test_json5.pl @@ -34,6 +34,7 @@ my @features = ( { name => 'trailing commas', file => 'json5_trailing_commas' }, { name => 'unquoted keys', file => 'json5_keys' }, { name => 'single-quoted strings', file => 'json5_strings' }, + { name => 'multi-line strings', file => 'json5_multiline' }, { name => 'numbers', file => 'json5_numbers' },); # Inputs that stay invalid even in json5 mode. @@ -191,6 +192,43 @@ foreach my $exe (@exes) } } + # Multi-line string continuations are also handled on the "not + # de-escaping" path (no -s flag), whole-input and byte-at-a-time. + my $ml_file = "$FindBin::RealBin/../json5_multiline.json5"; + foreach my $copt ([], [ "-c", 1 ]) + { + my ($stdout, $stderr) = + run_command([ @$exe, "--json5", @$copt, $ml_file ]); + + like($stdout, qr/SUCCESS/, + "json5 multi-line strings, no de-escaping (@$copt): parse succeeds" + ); + is($stderr, "", + "json5 multi-line strings, no de-escaping (@$copt): no error output" + ); + } + + # A backslash-newline continuation in a quoted key's value exercises + # the gate directly: a quoted key rules out the unrelated + # unquoted-key rejection reached via the fixture file. + my $cont_file = inline_file("{ \"a\": \"line \\\nb\" }"); + for my $size (64, 1) + { + my ($stdout, $stderr) = + run_command([ @$exe, "-s", "--json5", "-c", $size, $cont_file ]); + + is($stdout, "{\n\"a\": \"line b\"\n}", + "json5 mode, chunk size $size: backslash-newline continuation accepted" + ); + is($stderr, "", + "json5 mode, chunk size $size: backslash-newline continuation no error" + ); + + check_rejected($exe, $cont_file, + "non-json5 mode, chunk size $size: backslash-newline continuation", + qr/Escape sequence.*is invalid/s, "-c", $size); + } + # Specifically exercise a hex number split in the middle of its # digit run (0x1|F) and a leading-dot number split right after the # dot (.|5), at the chunk size that forces exactly that split. -- 2.43.0