From 1ec99d4af079e056dd4e2044ae49d1d9f9c3c51b Mon Sep 17 00:00:00 2001 From: po2xel Date: Tue, 19 Nov 2024 10:30:56 +0800 Subject: [PATCH 1/3] Update pipe.py Support date/datetime object in NumpyEncoder. --- choreographer/pipe.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/choreographer/pipe.py b/choreographer/pipe.py index 542b114b..0f1b4c39 100644 --- a/choreographer/pipe.py +++ b/choreographer/pipe.py @@ -29,6 +29,8 @@ def default(self, obj): return float(obj) elif hasattr(obj, "dtype") and obj.shape != (): return obj.tolist() + elif hasattr(obj, "isoformat"): + return obj.isoformat() return json.JSONEncoder.default(self, obj) From 5b758940e67a6314e0dddb3232676b6e57099efb Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Tue, 19 Nov 2024 11:16:35 -0500 Subject: [PATCH 2/3] Switch json to simplejson: Python's standard json encoder doesn't support NaN values (nan, +- inf) json doesn't, simplejson does to an extent. (all get converted to null) --- choreographer/pipe.py | 14 +++++++------- pyproject.toml | 3 +++ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/choreographer/pipe.py b/choreographer/pipe.py index 542b114b..fe24143b 100644 --- a/choreographer/pipe.py +++ b/choreographer/pipe.py @@ -1,6 +1,6 @@ import os import sys -import json +import simplejson import platform import warnings from threading import Lock @@ -11,7 +11,7 @@ class BlockWarning(UserWarning): # TODO: don't know about this # TODO: use has_attr instead of np.integer, you'll be fine -class NumpyEncoder(json.JSONEncoder): +class NumpyEncoder(simplejson.JSONEncoder): """Special json encoder for numpy types""" def default(self, obj): @@ -29,18 +29,18 @@ def default(self, obj): return float(obj) elif hasattr(obj, "dtype") and obj.shape != (): return obj.tolist() - return json.JSONEncoder.default(self, obj) + return simplejson.JSONEncoder.default(self, obj) class PipeClosedError(IOError): pass class Pipe: - def __init__(self, debug=False, cls=NumpyEncoder): + def __init__(self, debug=False, json_encoder=NumpyEncoder): self.read_from_chromium, self.write_from_chromium = list(os.pipe()) self.read_to_chromium, self.write_to_chromium = list(os.pipe()) self.debug = debug - self.cls=cls + self.json_encoder = json_encoder # this is just a convenience to prevent multiple shutdowns self.shutdown_lock = Lock() @@ -51,7 +51,7 @@ def write_json(self, obj, debug=None): if not debug: debug = self.debug if debug: print("write_json:", file=sys.stderr) - message = json.dumps(obj, ensure_ascii=False, cls=self.cls) + message = simplejson.dumps(obj, ensure_ascii=False, ignore_nan=True, cls=self.json_encoder) encoded_message = message.encode("utf-8") + b"\0" if debug: print(f"write_json: {message}", file=sys.stderr) @@ -112,7 +112,7 @@ def read_jsons(self, blocking=True, debug=None): for raw_message in decoded_buffer.split("\0"): if raw_message: try: - jsons.append(json.loads(raw_message)) + jsons.append(simplejson.loads(raw_message)) except BaseException as e: if debug: print(f"Problem with {raw_message} in json: {e}", file=sys.stderr) diff --git a/pyproject.toml b/pyproject.toml index b92a2f59..345140b4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,9 @@ authors = [ maintainers = [ {name = "Andrew Pikul", email = "ajpikul@gmail.com"}, ] +dependencies = [ + "simplejson" + ] [project.optional-dependencies] dev = [ From fa5ffc991dc2ffbca760413d289142d7a9193cf2 Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Tue, 19 Nov 2024 11:35:58 -0500 Subject: [PATCH 3/3] Change encoder name from numpy to multi --- choreographer/pipe.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/choreographer/pipe.py b/choreographer/pipe.py index a1c91aaf..fc49a29e 100644 --- a/choreographer/pipe.py +++ b/choreographer/pipe.py @@ -11,7 +11,7 @@ class BlockWarning(UserWarning): # TODO: don't know about this # TODO: use has_attr instead of np.integer, you'll be fine -class NumpyEncoder(simplejson.JSONEncoder): +class MultiEncoder(simplejson.JSONEncoder): """Special json encoder for numpy types""" def default(self, obj): @@ -38,7 +38,7 @@ class PipeClosedError(IOError): pass class Pipe: - def __init__(self, debug=False, json_encoder=NumpyEncoder): + def __init__(self, debug=False, json_encoder=MultiEncoder): self.read_from_chromium, self.write_from_chromium = list(os.pipe()) self.read_to_chromium, self.write_to_chromium = list(os.pipe()) self.debug = debug