From adcf9359569a938735e7bc3c0b527098c3bdff73 Mon Sep 17 00:00:00 2001 From: Leo Singer Date: Wed, 31 Jan 2024 11:08:39 -0500 Subject: [PATCH] Add black code formatter --- .flake8 | 3 + .github/workflows/build.yml | 2 + .vscode/settings.json | 5 ++ gcn_classic_to_kafka/cli.py | 44 +++++++---- gcn_classic_to_kafka/common.py | 4 +- gcn_classic_to_kafka/metrics.py | 29 +++---- gcn_classic_to_kafka/socket.py | 49 ++++++------ gcn_classic_to_kafka/test/test_socket.py | 24 +++--- poetry.lock | 98 +++++++++++++++++++++++- pyproject.toml | 1 + 10 files changed, 193 insertions(+), 66 deletions(-) create mode 100644 .flake8 create mode 100644 .vscode/settings.json diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..f295e07 --- /dev/null +++ b/.flake8 @@ -0,0 +1,3 @@ +[flake8] +max-line-length = 88 +extend-ignore = E203, E704 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 99d42ac..638ffd2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -29,6 +29,8 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 + - uses: psf/black@stable + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..3775516 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,5 @@ +{ + "editor.defaultFormatter": "ms-python.black-formatter", + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true +} diff --git a/gcn_classic_to_kafka/cli.py b/gcn_classic_to_kafka/cli.py index 3808f5b..5964552 100644 --- a/gcn_classic_to_kafka/cli.py +++ b/gcn_classic_to_kafka/cli.py @@ -23,7 +23,7 @@ def signal_handler(signum, frame): - log.info('Exiting due to signal %d', signum) + log.info("Exiting due to signal %d", signum) sys.exit(128 + signum) @@ -36,19 +36,31 @@ def host_port(host_port_str): # Parse netloc like it is done for HTTP URLs. # This ensures that we will get the correct behavior for hostname:port # splitting even for IPv6 addresses. - return urllib.parse.urlparse(f'http://{host_port_str}') + return urllib.parse.urlparse(f"http://{host_port_str}") @click.command() @click.option( - '--listen', type=host_port, default=':8081', show_default=True, - help='Hostname and port to listen on for GCN Classic') + "--listen", + type=host_port, + default=":8081", + show_default=True, + help="Hostname and port to listen on for GCN Classic", +) @click.option( - '--prometheus', type=host_port, default=':8000', show_default=True, - help='Hostname and port to listen on for Prometheus metric reporting') + "--prometheus", + type=host_port, + default=":8000", + show_default=True, + help="Hostname and port to listen on for Prometheus metric reporting", +) @click.option( - '--loglevel', type=click.Choice(logging._levelToName.values()), - default='DEBUG', show_default=True, help='Log level') + "--loglevel", + type=click.Choice(logging._levelToName.values()), + default="DEBUG", + show_default=True, + help="Log level", +) def main(listen, prometheus, loglevel): """Pump GCN Classic notices to a Kafka broker. @@ -64,21 +76,21 @@ def main(listen, prometheus, loglevel): """ logging.basicConfig(level=loglevel) - prometheus_client.start_http_server(prometheus.port, - prometheus.hostname or '0.0.0.0') - log.info('Prometheus listening on %s', prometheus.netloc) + prometheus_client.start_http_server( + prometheus.port, prometheus.hostname or "0.0.0.0" + ) + log.info("Prometheus listening on %s", prometheus.netloc) config = gcn_kafka.config_from_env() - config['client.id'] = __package__ - config['on_delivery'] = kafka_delivered_cb + config["client.id"] = __package__ + config["on_delivery"] = kafka_delivered_cb producer = gcn_kafka.Producer(config) client = client_connected(producer) async def serve(): - server = await asyncio.start_server( - client, listen.hostname, listen.port) - log.info('GCN server listening on %s', listen.netloc) + server = await asyncio.start_server(client, listen.hostname, listen.port) + log.info("GCN server listening on %s", listen.netloc) async with server: await server.serve_forever() diff --git a/gcn_classic_to_kafka/common.py b/gcn_classic_to_kafka/common.py index d77aa3b..9ffb2bc 100644 --- a/gcn_classic_to_kafka/common.py +++ b/gcn_classic_to_kafka/common.py @@ -38,7 +38,7 @@ def notice_type_int_to_str(notice_type_int): try: notice_type_enum = gcn.NoticeType(notice_type_int) except ValueError: - notice_type_str = 'UNKNOWN' + notice_type_str = "UNKNOWN" else: notice_type_str = notice_type_enum.name return notice_type_str @@ -66,4 +66,4 @@ def topic_for_notice_type_str(notice_type_str, flavor): 'gcn.classic.voevent.LVC_PRELIMINARY' """ - return f'gcn.classic.{flavor}.{notice_type_str}' + return f"gcn.classic.{flavor}.{notice_type_str}" diff --git a/gcn_classic_to_kafka/metrics.py b/gcn_classic_to_kafka/metrics.py index 6d92719..759becb 100644 --- a/gcn_classic_to_kafka/metrics.py +++ b/gcn_classic_to_kafka/metrics.py @@ -9,22 +9,25 @@ import prometheus_client connected = prometheus_client.Gauge( - 'connected', 'Number of active connections from GCN Classic', - namespace=__package__) + "connected", "Number of active connections from GCN Classic", namespace=__package__ +) iamalive = prometheus_client.Counter( - 'iamalive', - 'GCN notices received of any type, including iamalives', - namespace=__package__) + "iamalive", + "GCN notices received of any type, including iamalives", + namespace=__package__, +) received = prometheus_client.Counter( - 'received', - 'GCN Classic notices received by notice type and flavor', - labelnames=['notice_type_int', 'notice_type_str', 'flavor'], - namespace=__package__) + "received", + "GCN Classic notices received by notice type and flavor", + labelnames=["notice_type_int", "notice_type_str", "flavor"], + namespace=__package__, +) delivered = prometheus_client.Counter( - 'delivered', - 'Kafka messages delivered', - labelnames=['topic', 'partition', 'successful'], - namespace=__package__) + "delivered", + "Kafka messages delivered", + labelnames=["topic", "partition", "successful"], + namespace=__package__, +) diff --git a/gcn_classic_to_kafka/socket.py b/gcn_classic_to_kafka/socket.py index 818a1d7..0731355 100644 --- a/gcn_classic_to_kafka/socket.py +++ b/gcn_classic_to_kafka/socket.py @@ -20,32 +20,34 @@ log = logging.getLogger(__name__) bin_len = 160 -int4 = struct.Struct('!l') -ignore_notice_types = {gcn.NoticeType.IM_ALIVE, - gcn.NoticeType.VOE_11_IM_ALIVE, - gcn.NoticeType.VOE_20_IM_ALIVE} +int4 = struct.Struct("!l") +ignore_notice_types = { + gcn.NoticeType.IM_ALIVE, + gcn.NoticeType.VOE_11_IM_ALIVE, + gcn.NoticeType.VOE_20_IM_ALIVE, +} def client_connected(producer: confluent_kafka.Producer, timeout: float = 90): - async def client_connected_cb(reader: asyncio.StreamReader, - writer: asyncio.StreamWriter): + async def client_connected_cb( + reader: asyncio.StreamReader, writer: asyncio.StreamWriter + ): async def read(): bin_data = await reader.readexactly(bin_len) - voe_len, = int4.unpack(await reader.readexactly(int4.size)) + (voe_len,) = int4.unpack(await reader.readexactly(int4.size)) voe_data = await reader.readexactly(voe_len) - txt_len, = int4.unpack(await reader.readexactly(int4.size)) + (txt_len,) = int4.unpack(await reader.readexactly(int4.size)) txt_data = await reader.readexactly(txt_len) - log.debug('Read %d + %d + %d bytes', bin_len, voe_len, txt_len) + log.debug("Read %d + %d + %d bytes", bin_len, voe_len, txt_len) return bin_data, voe_data, txt_data async def process(): - bin_data, voe_data, txt_data = await asyncio.wait_for( - read(), timeout) + bin_data, voe_data, txt_data = await asyncio.wait_for(read(), timeout) metrics.iamalive.inc() - bin_notice_type, = int4.unpack_from(bin_data) - log.info('Received notice of type 0x%08X', bin_notice_type) + (bin_notice_type,) = int4.unpack_from(bin_data) + log.info("Received notice of type 0x%08X", bin_notice_type) if bin_notice_type in ignore_notice_types: return @@ -54,20 +56,21 @@ async def process(): if bin_notice_type != voe_notice_type: log.warning( - 'Binary (0x%08X) and VOEvent (0x%08X) notice types differ', - bin_notice_type, voe_notice_type) + "Binary (0x%08X) and VOEvent (0x%08X) notice types differ", + bin_notice_type, + voe_notice_type, + ) # The text notices do not contain a machine-readable notice type. txt_notice_type = bin_notice_type for notice_type_int, data, flavor in [ - [bin_notice_type, bin_data, 'binary'], - [voe_notice_type, voe_data, 'voevent'], - [txt_notice_type, txt_data, 'text'], + [bin_notice_type, bin_data, "binary"], + [voe_notice_type, voe_data, "voevent"], + [txt_notice_type, txt_data, "text"], ]: notice_type_str = notice_type_int_to_str(notice_type_int) - metrics.received.labels( - notice_type_int, notice_type_str, flavor).inc() + metrics.received.labels(notice_type_int, notice_type_str, flavor).inc() topic = topic_for_notice_type_str(notice_type_str, flavor) producer.produce(topic, data) @@ -75,14 +78,14 @@ async def process(): # report callbacks to be triggered. producer.poll(0) - peer, *_ = writer.get_extra_info('peername') - log.info('Client connected from %s', peer) + peer, *_ = writer.get_extra_info("peername") + log.info("Client connected from %s", peer) try: with metrics.connected.track_inprogress(): while True: await process() finally: - log.info('Closing connection from %s', peer) + log.info("Closing connection from %s", peer) writer.close() await writer.wait_closed() diff --git a/gcn_classic_to_kafka/test/test_socket.py b/gcn_classic_to_kafka/test/test_socket.py index acc7d96..7bd7109 100644 --- a/gcn_classic_to_kafka/test/test_socket.py +++ b/gcn_classic_to_kafka/test/test_socket.py @@ -34,21 +34,24 @@ def make_packet(bin_notice_type, voe_notice_type=None): """.encode() - text = b'Hello world' + text = b"Hello world" return ( - struct.pack('!l156xl', bin_notice_type, len(voevent)) + voevent + - struct.pack('!l', len(text)) + text) + struct.pack("!l156xl", bin_notice_type, len(voevent)) + + voevent + + struct.pack("!l", len(text)) + + text + ) @pytest_asyncio.fixture async def start_server(): producer = mock.Mock() cb = client_connected(producer=producer, timeout=timeout) - server = await asyncio.start_server(cb, '127.0.0.1') + server = await asyncio.start_server(cb, "127.0.0.1") async with server: server_task = asyncio.create_task(server.serve_forever()) - socket, = server.sockets + (socket,) = server.sockets host, port = socket.getsockname() yield producer, host, port server_task.cancel() @@ -96,10 +99,13 @@ async def test_socket(start_server): await writer.drain() await asyncio.sleep(0.25 * timeout) assert not reader.at_eof() - producer.produce.assert_has_calls([ - mock.call('gcn.classic.binary.LVC_TEST', mock.ANY), - mock.call('gcn.classic.voevent.LVC_TEST', mock.ANY), - mock.call('gcn.classic.text.LVC_TEST', mock.ANY)]) + producer.produce.assert_has_calls( + [ + mock.call("gcn.classic.binary.LVC_TEST", mock.ANY), + mock.call("gcn.classic.voevent.LVC_TEST", mock.ANY), + mock.call("gcn.classic.text.LVC_TEST", mock.ANY), + ] + ) writer.close() await writer.wait_closed() diff --git a/poetry.lock b/poetry.lock index c1d1c64..f5d7f43 100644 --- a/poetry.lock +++ b/poetry.lock @@ -14,6 +14,52 @@ files = [ [package.dependencies] cryptography = ">=3.2" +[[package]] +name = "black" +version = "24.1.1" +description = "The uncompromising code formatter." +optional = false +python-versions = ">=3.8" +files = [ + {file = "black-24.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2588021038bd5ada078de606f2a804cadd0a3cc6a79cb3e9bb3a8bf581325a4c"}, + {file = "black-24.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1a95915c98d6e32ca43809d46d932e2abc5f1f7d582ffbe65a5b4d1588af7445"}, + {file = "black-24.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2fa6a0e965779c8f2afb286f9ef798df770ba2b6cee063c650b96adec22c056a"}, + {file = "black-24.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:5242ecd9e990aeb995b6d03dc3b2d112d4a78f2083e5a8e86d566340ae80fec4"}, + {file = "black-24.1.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:fc1ec9aa6f4d98d022101e015261c056ddebe3da6a8ccfc2c792cbe0349d48b7"}, + {file = "black-24.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0269dfdea12442022e88043d2910429bed717b2d04523867a85dacce535916b8"}, + {file = "black-24.1.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b3d64db762eae4a5ce04b6e3dd745dcca0fb9560eb931a5be97472e38652a161"}, + {file = "black-24.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:5d7b06ea8816cbd4becfe5f70accae953c53c0e53aa98730ceccb0395520ee5d"}, + {file = "black-24.1.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:e2c8dfa14677f90d976f68e0c923947ae68fa3961d61ee30976c388adc0b02c8"}, + {file = "black-24.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a21725862d0e855ae05da1dd25e3825ed712eaaccef6b03017fe0853a01aa45e"}, + {file = "black-24.1.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:07204d078e25327aad9ed2c64790d681238686bce254c910de640c7cc4fc3aa6"}, + {file = "black-24.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:a83fe522d9698d8f9a101b860b1ee154c1d25f8a82ceb807d319f085b2627c5b"}, + {file = "black-24.1.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:08b34e85170d368c37ca7bf81cf67ac863c9d1963b2c1780c39102187ec8dd62"}, + {file = "black-24.1.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7258c27115c1e3b5de9ac6c4f9957e3ee2c02c0b39222a24dc7aa03ba0e986f5"}, + {file = "black-24.1.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:40657e1b78212d582a0edecafef133cf1dd02e6677f539b669db4746150d38f6"}, + {file = "black-24.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:e298d588744efda02379521a19639ebcd314fba7a49be22136204d7ed1782717"}, + {file = "black-24.1.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:34afe9da5056aa123b8bfda1664bfe6fb4e9c6f311d8e4a6eb089da9a9173bf9"}, + {file = "black-24.1.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:854c06fb86fd854140f37fb24dbf10621f5dab9e3b0c29a690ba595e3d543024"}, + {file = "black-24.1.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3897ae5a21ca132efa219c029cce5e6bfc9c3d34ed7e892113d199c0b1b444a2"}, + {file = "black-24.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:ecba2a15dfb2d97105be74bbfe5128bc5e9fa8477d8c46766505c1dda5883aac"}, + {file = "black-24.1.1-py3-none-any.whl", hash = "sha256:5cdc2e2195212208fbcae579b931407c1fa9997584f0a415421748aeafff1168"}, + {file = "black-24.1.1.tar.gz", hash = "sha256:48b5760dcbfe5cf97fd4fba23946681f3a81514c6ab8a45b50da67ac8fbc6c7b"}, +] + +[package.dependencies] +click = ">=8.0.0" +mypy-extensions = ">=0.4.3" +packaging = ">=22.0" +pathspec = ">=0.9.0" +platformdirs = ">=2" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.0.1", markers = "python_version < \"3.11\""} + +[package.extras] +colorama = ["colorama (>=0.4.3)"] +d = ["aiohttp (>=3.7.4)", "aiohttp (>=3.7.4,!=3.9.0)"] +jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] +uvloop = ["uvloop (>=0.15.2)"] + [[package]] name = "certifi" version = "2023.7.22" @@ -458,8 +504,6 @@ files = [ {file = "lxml-4.9.3-cp27-cp27m-macosx_11_0_x86_64.whl", hash = "sha256:b0a545b46b526d418eb91754565ba5b63b1c0b12f9bd2f808c852d9b4b2f9b5c"}, {file = "lxml-4.9.3-cp27-cp27m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:075b731ddd9e7f68ad24c635374211376aa05a281673ede86cbe1d1b3455279d"}, {file = "lxml-4.9.3-cp27-cp27m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:1e224d5755dba2f4a9498e150c43792392ac9b5380aa1b845f98a1618c94eeef"}, - {file = "lxml-4.9.3-cp27-cp27m-win32.whl", hash = "sha256:2c74524e179f2ad6d2a4f7caf70e2d96639c0954c943ad601a9e146c76408ed7"}, - {file = "lxml-4.9.3-cp27-cp27m-win_amd64.whl", hash = "sha256:4f1026bc732b6a7f96369f7bfe1a4f2290fb34dce00d8644bc3036fb351a4ca1"}, {file = "lxml-4.9.3-cp27-cp27mu-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c0781a98ff5e6586926293e59480b64ddd46282953203c76ae15dbbbf302e8bb"}, {file = "lxml-4.9.3-cp27-cp27mu-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:cef2502e7e8a96fe5ad686d60b49e1ab03e438bd9123987994528febd569868e"}, {file = "lxml-4.9.3-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:b86164d2cff4d3aaa1f04a14685cbc072efd0b4f99ca5708b2ad1b9b5988a991"}, @@ -566,6 +610,17 @@ files = [ {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"}, ] +[[package]] +name = "mypy-extensions" +version = "1.0.0" +description = "Type system extensions for programs checked with the mypy type checker." +optional = false +python-versions = ">=3.5" +files = [ + {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, + {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, +] + [[package]] name = "packaging" version = "23.1" @@ -577,6 +632,32 @@ files = [ {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, ] +[[package]] +name = "pathspec" +version = "0.12.1" +description = "Utility library for gitignore style pattern matching of file paths." +optional = false +python-versions = ">=3.8" +files = [ + {file = "pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08"}, + {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"}, +] + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + [[package]] name = "pluggy" version = "1.2.0" @@ -743,6 +824,17 @@ files = [ {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + [[package]] name = "urllib3" version = "2.0.7" @@ -763,4 +855,4 @@ zstd = ["zstandard (>=0.18.0)"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "9883027a4d3dbf07075a9a30375e4e1a3aa469cf40cb6e9be6375359ae0d96b9" +content-hash = "39fcca480ad0fcbbe30f66f05242dafcd207df647f516022bc7bbbe2ea1401a3" diff --git a/pyproject.toml b/pyproject.toml index e53c695..3e2bb8c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,7 @@ pygcn = "*" prometheus-client = "^0.17.0" [tool.poetry.group.dev.dependencies] +black = "*" flake8 = "*" pytest = "*" pytest-asyncio = "*"