Reduce overhead to save json data to postgresql (#88717)

* Reduce overhead to strip nulls from json

* Reduce overhead to strip nulls from json

* small cleanup
pull/88738/head
J. Nick Koston 2023-02-24 20:37:36 -06:00 committed by GitHub
parent 7b2e743a6b
commit 0223058d25
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 16 additions and 26 deletions

View File

@ -83,38 +83,28 @@ def json_bytes(data: Any) -> bytes:
)
def json_bytes_strip_null(data: Any) -> bytes:
"""Dump json bytes after terminating strings at the first NUL."""
def process_dict(_dict: dict[Any, Any]) -> dict[Any, Any]:
"""Strip NUL from items in a dict."""
return {key: strip_null(o) for key, o in _dict.items()}
def process_list(_list: list[Any]) -> list[Any]:
"""Strip NUL from items in a list."""
return [strip_null(o) for o in _list]
def strip_null(obj: Any) -> Any:
def _strip_null(obj: Any) -> Any:
"""Strip NUL from an object."""
if isinstance(obj, str):
return obj.split("\0", 1)[0]
if isinstance(obj, dict):
return process_dict(obj)
return {key: _strip_null(o) for key, o in obj.items()}
if isinstance(obj, list):
return process_list(obj)
return [_strip_null(o) for o in obj]
return obj
def json_bytes_strip_null(data: Any) -> bytes:
"""Dump json bytes after terminating strings at the first NUL."""
# We expect null-characters to be very rare, hence try encoding first and look
# for an escaped null-character in the output.
result = json_bytes(data)
if b"\\u0000" in result:
if b"\\u0000" not in result:
return result
# We work on the processed result so we don't need to worry about
# Home Assistant extensions which allows encoding sets, tuples, etc.
data_processed = orjson.loads(result)
data_processed = strip_null(data_processed)
result = json_bytes(data_processed)
return result
return json_bytes(_strip_null(orjson.loads(result)))
def json_dumps(data: Any) -> str: