diff --git a/src/partial_json_parser/core/complete.py b/src/partial_json_parser/core/complete.py index ed9b99e..44bcf8e 100644 --- a/src/partial_json_parser/core/complete.py +++ b/src/partial_json_parser/core/complete.py @@ -17,7 +17,8 @@ def fix(json_string: str, allow_partial: Union[Allow, int] = ALL): def _fix(json_string: str, allow: Allow, is_top_level=False): try: - result = complete_any(json_string.strip(), allow, is_top_level) + # Trailing whitespace can belong to an unfinished string. + result = complete_any(json_string.lstrip(), allow, is_top_level) if result is False: raise PartialJSON @@ -45,7 +46,7 @@ def complete_any(json_string: str, allow: Allow, is_top_level=False) -> Complete return complete_str(json_string, allow) if char in "1234567890": - return complete_num(json_string, allow, is_top_level) + return complete_num(json_string.rstrip(), allow, is_top_level) if char == "[": return complete_arr(json_string, allow) @@ -53,6 +54,9 @@ def complete_any(json_string: str, allow: Allow, is_top_level=False) -> Complete if char == "{": return complete_obj(json_string, allow) + # Literal recovery does not need to preserve string content. + json_string = json_string.rstrip() + if json_string.startswith("null"): return (4, True) if "null".startswith(json_string): @@ -122,18 +126,27 @@ def not_escaped(index: int): count = index - len(text_before.rstrip("\\")) return count % 2 == 0 + # Keep the existing recovery of incomplete escapes and control suffixes. + trimmed_end = len(json_string.rstrip()) + escape_end = min(i, trimmed_end) + if escape_end and json_string[escape_end - 1] == "\\" and not_escaped(escape_end - 1): + escape_end -= 1 + i = escape_end + elif any(ord(char) < 32 for char in json_string[trimmed_end:]): + i = trimmed_end + # \uXXXX - _u = json_string.rfind("\\u", max(0, i - 5), i) + _u = json_string.rfind("\\u", max(0, escape_end - 5), escape_end) if _u != -1 and not_escaped(_u): return _u, '"' # \UXXXXXXXX - _U = json_string.rfind("\\U", max(0, i - 9), i) + _U = json_string.rfind("\\U", max(0, escape_end - 9), escape_end) if _U != -1 and not_escaped(_U): return _U, '"' # \xXX - _x = json_string.rfind("\\x", max(0, i - 3), i) + _x = json_string.rfind("\\x", max(0, escape_end - 3), escape_end) if _x != -1 and not_escaped(_x): return _x, '"' diff --git a/tests/test_examples.py b/tests/test_examples.py index 7bf8727..75389b9 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -1,10 +1,11 @@ from itertools import accumulate from json import dumps +from json import loads as standard_loads from math import isnan from hypothesis import given, settings from hypothesis.strategies import integers -from pytest import raises +from pytest import mark, raises from test_hypotheses import json from partial_json_parser import * @@ -139,3 +140,97 @@ def test_consistency(): def test_consistencies(json_string, allow): for json_string in accumulate(json_string): assert consistent(json_string, allow), f"{Allow(allow)!r} - {json_string}" + + +@mark.parametrize("use_fast_fix", [True, False]) +@mark.parametrize( + "text, expected", + [ + ('"hello ', "hello "), + ('" ', " "), + ('"hello\u00a0', "hello\u00a0"), + ('"hello\u3000', "hello\u3000"), + ('["hello ', ["hello "]), + ('[0, "hello ', [0, "hello "]), + ('{"key": "hello ', {"key": "hello "}), + ('{"ok": true, "key": "hello ', {"ok": True, "key": "hello "}), + ('[{"key": "hello ', [{"key": "hello "}]), + ], +) +def test_partial_string_keeps_trailing_whitespace(text, expected, use_fast_fix): + assert parse_json(text, use_fast_fix=use_fast_fix) == expected + + +@mark.parametrize("use_fast_fix", [True, False]) +@mark.parametrize("text", ['"hello "', '["hello "]', '{"key": "hello "}']) +def test_complete_string_with_trailing_json_whitespace(text, use_fast_fix): + text += " \t\r\n" + assert parse_json(text, use_fast_fix=use_fast_fix) == standard_loads(text) + + +@mark.parametrize("use_fast_fix", [True, False]) +@mark.parametrize("text, allow, expected", [('["hello ', ARR, []), ('{"key": "hello ', OBJ, {})]) +def test_string_whitespace_does_not_enable_partial_strings(text, allow, expected, use_fast_fix): + assert parse_json(text, allow, use_fast_fix=use_fast_fix) == expected + + +@mark.parametrize("use_fast_fix", [True, False]) +@mark.parametrize("container", ["{text}", "[{text}", '{{"key": {text}']) +@mark.parametrize("prefix, expected", [("t", True), ("f", False), ("n", None), ("Inf", float("inf")), ("-Inf", float("-inf")), ("Na", float("nan"))]) +def test_partial_literal_keeps_whitespace_recovery(prefix, expected, container, use_fast_fix): + value = parse_json(container.format(text=prefix + " \t\r\n"), use_fast_fix=use_fast_fix) + if container.startswith("["): + value = value[0] + elif container.startswith("{{"): + value = value["key"] + if isinstance(expected, float) and isnan(expected): + assert isnan(value) + else: + assert value == expected + + +@mark.parametrize("use_fast_fix", [True, False]) +@mark.parametrize("container", ["{text}", "[{text}", '{{"key": {text}']) +@mark.parametrize( + "text, expected", + [ + ('"hello\t', "hello"), + ('"hello\n ', "hello"), + ('"hello\\ ', "hello"), + (r'"\u0 ', ""), + (r'"\U0 ', ""), + (r'"\x0 ', ""), + (r'"\u000\ ', ""), + ], +) +def test_partial_string_keeps_invalid_suffix_recovery(text, expected, container, use_fast_fix): + value = parse_json(container.format(text=text), use_fast_fix=use_fast_fix) + if container.startswith("["): + value = value[0] + elif container.startswith("{{"): + value = value["key"] + assert value == expected + + +@mark.parametrize("use_fast_fix", [True, False]) +@mark.parametrize( + "text, expected", + [ + (r'"hello\t ', "hello\t "), + (r'"hello\n ', "hello\n "), + (r'"\u0061 ', "a "), + (r'"\\u0 ', "\\u0 "), + (r'"\\x0 ', "\\x0 "), + (r'"hello\\ ', "hello\\ "), + ], +) +def test_partial_string_keeps_spaces_after_valid_escapes(text, expected, use_fast_fix): + assert parse_json(text, use_fast_fix=use_fast_fix) == expected + + +@mark.parametrize("use_fast_fix", [True, False]) +@mark.parametrize("number", ["1", "-1", "1e+2", "-1e+2"]) +@mark.parametrize("container, allow, expected", [("[{text}", ARR, []), ('{{"key": {text}', OBJ, {})]) +def test_numeric_whitespace_keeps_partial_restrictions(number, container, allow, expected, use_fast_fix): + text = container.format(text=number + " \t\n") + assert parse_json(text, allow, use_fast_fix=use_fast_fix) == expected