From 506942b6d018e3ff3a250d6181a4cd40a744a32a Mon Sep 17 00:00:00 2001 From: Saejin Mahlau-Heinert Date: Mon, 19 Jun 2023 22:29:42 -0500 Subject: [PATCH 01/32] Initial incomplete python implementation --- Dockerfile | 15 ++++++++ microwrap/__init__.py | 2 ++ microwrap/microwrap.py | 82 ++++++++++++++++++++++++++++++++++++++++++ pyproject.toml | 23 ++++++++++++ 4 files changed, 122 insertions(+) create mode 100644 microwrap/__init__.py create mode 100644 microwrap/microwrap.py create mode 100644 pyproject.toml diff --git a/Dockerfile b/Dockerfile index e69de29..3e2f283 100644 --- a/Dockerfile +++ b/Dockerfile @@ -0,0 +1,15 @@ +# Build microwrap +FROM alpine:3.18 as build + +# Install dependencies +RUN apk add --no-cache poetry +RUN poetry install +RUN poetry run task compile + +# Create microwrap image +FROM alpine:3.18 + +COPY --from=build /microwrap /usr/bin/microwrap +COPY --from=build /usr/local/lib/libmicrohttpd.so.12 /usr/local/lib + +ENTRYPOINT [ "microwrap" ] diff --git a/microwrap/__init__.py b/microwrap/__init__.py new file mode 100644 index 0000000..4acb153 --- /dev/null +++ b/microwrap/__init__.py @@ -0,0 +1,2 @@ +"""MicroWrap translates HTTP requests to invocations of an arbitrary executable.""" +from microwrap import microwrap as application diff --git a/microwrap/microwrap.py b/microwrap/microwrap.py new file mode 100644 index 0000000..1e91892 --- /dev/null +++ b/microwrap/microwrap.py @@ -0,0 +1,82 @@ +"""Main MicroWrap module.""" +import sys +import json +from urllib.parse import parse_qs +import time + +from typing import Any, Iterable +from wsgiref.types import WSGIEnvironment, StartResponse + +# this can be replaced with any WSGI server +import bjoern as wsgi_server + +USAGE_HELP = "Usage: microwrap " +CONFIG_PATH = "/microwrap.json" +RESPONSE_HEADERS = [] + + +def load_config() -> dict[str, Any]: + """Load configuration from disk.""" + with open(CONFIG_PATH, encoding="utf-8") as file: + return json.load(file) + + +def execute_command(config: dict[str, Any], params: dict[str, list[str]]): + """Build the command to execute.""" + time.sleep(2) + + executable = config.get("executablePath", "") + options = config.get("defaultParameters", {}) + for key, value in params.items(): + if config.get("allowedParameters").contains(key): + options[key] = value + + arguments = [] + + for key, value in options.items(): + if value == False or value == True or value == "": + arguments.append(f"--{key}") + else: + arguments.append(f"--{key} {value}") + + # execute command + + # return output + return executable + " " + " ".join(arguments) + + +def microwrap(env: WSGIEnvironment, start_response: StartResponse) -> Iterable[bytes]: + """WSGI application that translates HTTP requests to invocations of an arbitrary executable.""" + # input_body = environ["wsgi.input"].read() + # errors = env["wsgi.errors"] + config = load_config() + method = env.get("REQUEST_METHOD", "") + path = env.get("SCRIPT_NAME", "") + env.get("PATH_TRANSLATED", "") + params = parse_qs(env.get("QUERY_STRING", ""), keep_blank_values=True) + + print(f"[{method}][{path}] Configuration: {config}\nParameters: {params}") + + response_body = execute_command(config, params) + status = "200 OK" + start_response(status, RESPONSE_HEADERS) + return [response_body.encode()] + + +def run(host="localhost", port=80): + """Run the WSGI application.""" + print(f"Listening on {host}:{port}...") + wsgi_server.run(microwrap, host, port) + + +if __name__ == "__main__": + if len(sys.argv == 1): + run() + elif len(sys.argv) == 2: + if sys.argv[1] == "--help" or sys.argv[1] == "-h": + print(USAGE_HELP) + else: + run(sys.argv[1]) + elif len(sys.argv) == 3: + run(sys.argv[1], sys.argv[2]) + else: + print(USAGE_HELP) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..2fa01dd --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,23 @@ +[tool.poetry] +name = "microwrap" +version = "1.0.0" +description = "Translate HTTP requests to invocations of an arbitrary executable" +authors = ["Saejin Mahlau-Heinert "] +license = "UNLICENSED" +readme = "README.md" + +[tool.poetry.dependencies] +python = "==3.11.3" +bjoern = "^3.2.2" + +[tool.poetry.group.dev.dependencies] +black = "^23.3.0" +pylint = "^2.17.4" +taskipy = "^1.11.0" + +[tool.taskipy.tasks] +lint = { cmd = "pylint microwrap", help = "check code style with pylint" } + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" From d5c611b0125a90acafc7b23c85077e96e2faeede Mon Sep 17 00:00:00 2001 From: Saejin Mahlau-Heinert Date: Mon, 19 Jun 2023 22:29:59 -0500 Subject: [PATCH 02/32] Add in-progress work on cythonization --- poetry.lock | 472 +++++++++++++++++++++++++++++++++++++++++++++++++ pyproject.toml | 4 + 2 files changed, 476 insertions(+) create mode 100644 poetry.lock diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 0000000..a6140be --- /dev/null +++ b/poetry.lock @@ -0,0 +1,472 @@ +# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. + +[[package]] +name = "astroid" +version = "2.15.5" +description = "An abstract syntax tree for Python with inference support." +optional = false +python-versions = ">=3.7.2" +files = [ + {file = "astroid-2.15.5-py3-none-any.whl", hash = "sha256:078e5212f9885fa85fbb0cf0101978a336190aadea6e13305409d099f71b2324"}, + {file = "astroid-2.15.5.tar.gz", hash = "sha256:1039262575027b441137ab4a62a793a9b43defb42c32d5670f38686207cd780f"}, +] + +[package.dependencies] +lazy-object-proxy = ">=1.4.0" +wrapt = {version = ">=1.14,<2", markers = "python_version >= \"3.11\""} + +[[package]] +name = "bjoern" +version = "3.2.2" +description = "A screamingly fast Python 2 + 3 WSGI server written in C." +optional = false +python-versions = "*" +files = [ + {file = "bjoern-3.2.2.tar.gz", hash = "sha256:16e5a02a9a17a7f5f8bea0d7c58650e78ab80ead6fe3e390037573d4355baf31"}, +] + +[[package]] +name = "black" +version = "23.3.0" +description = "The uncompromising code formatter." +optional = false +python-versions = ">=3.7" +files = [ + {file = "black-23.3.0-cp310-cp310-macosx_10_16_arm64.whl", hash = "sha256:0945e13506be58bf7db93ee5853243eb368ace1c08a24c65ce108986eac65915"}, + {file = "black-23.3.0-cp310-cp310-macosx_10_16_universal2.whl", hash = "sha256:67de8d0c209eb5b330cce2469503de11bca4085880d62f1628bd9972cc3366b9"}, + {file = "black-23.3.0-cp310-cp310-macosx_10_16_x86_64.whl", hash = "sha256:7c3eb7cea23904399866c55826b31c1f55bbcd3890ce22ff70466b907b6775c2"}, + {file = "black-23.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:32daa9783106c28815d05b724238e30718f34155653d4d6e125dc7daec8e260c"}, + {file = "black-23.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:35d1381d7a22cc5b2be2f72c7dfdae4072a3336060635718cc7e1ede24221d6c"}, + {file = "black-23.3.0-cp311-cp311-macosx_10_16_arm64.whl", hash = "sha256:a8a968125d0a6a404842fa1bf0b349a568634f856aa08ffaff40ae0dfa52e7c6"}, + {file = "black-23.3.0-cp311-cp311-macosx_10_16_universal2.whl", hash = "sha256:c7ab5790333c448903c4b721b59c0d80b11fe5e9803d8703e84dcb8da56fec1b"}, + {file = "black-23.3.0-cp311-cp311-macosx_10_16_x86_64.whl", hash = "sha256:a6f6886c9869d4daae2d1715ce34a19bbc4b95006d20ed785ca00fa03cba312d"}, + {file = "black-23.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f3c333ea1dd6771b2d3777482429864f8e258899f6ff05826c3a4fcc5ce3f70"}, + {file = "black-23.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:11c410f71b876f961d1de77b9699ad19f939094c3a677323f43d7a29855fe326"}, + {file = "black-23.3.0-cp37-cp37m-macosx_10_16_x86_64.whl", hash = "sha256:1d06691f1eb8de91cd1b322f21e3bfc9efe0c7ca1f0e1eb1db44ea367dff656b"}, + {file = "black-23.3.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:50cb33cac881766a5cd9913e10ff75b1e8eb71babf4c7104f2e9c52da1fb7de2"}, + {file = "black-23.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:e114420bf26b90d4b9daa597351337762b63039752bdf72bf361364c1aa05925"}, + {file = "black-23.3.0-cp38-cp38-macosx_10_16_arm64.whl", hash = "sha256:48f9d345675bb7fbc3dd85821b12487e1b9a75242028adad0333ce36ed2a6d27"}, + {file = "black-23.3.0-cp38-cp38-macosx_10_16_universal2.whl", hash = "sha256:714290490c18fb0126baa0fca0a54ee795f7502b44177e1ce7624ba1c00f2331"}, + {file = "black-23.3.0-cp38-cp38-macosx_10_16_x86_64.whl", hash = "sha256:064101748afa12ad2291c2b91c960be28b817c0c7eaa35bec09cc63aa56493c5"}, + {file = "black-23.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:562bd3a70495facf56814293149e51aa1be9931567474993c7942ff7d3533961"}, + {file = "black-23.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:e198cf27888ad6f4ff331ca1c48ffc038848ea9f031a3b40ba36aced7e22f2c8"}, + {file = "black-23.3.0-cp39-cp39-macosx_10_16_arm64.whl", hash = "sha256:3238f2aacf827d18d26db07524e44741233ae09a584273aa059066d644ca7b30"}, + {file = "black-23.3.0-cp39-cp39-macosx_10_16_universal2.whl", hash = "sha256:f0bd2f4a58d6666500542b26354978218a9babcdc972722f4bf90779524515f3"}, + {file = "black-23.3.0-cp39-cp39-macosx_10_16_x86_64.whl", hash = "sha256:92c543f6854c28a3c7f39f4d9b7694f9a6eb9d3c5e2ece488c327b6e7ea9b266"}, + {file = "black-23.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a150542a204124ed00683f0db1f5cf1c2aaaa9cc3495b7a3b5976fb136090ab"}, + {file = "black-23.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:6b39abdfb402002b8a7d030ccc85cf5afff64ee90fa4c5aebc531e3ad0175ddb"}, + {file = "black-23.3.0-py3-none-any.whl", hash = "sha256:ec751418022185b0c1bb7d7736e6933d40bbb14c14a0abcf9123d1b159f98dd4"}, + {file = "black-23.3.0.tar.gz", hash = "sha256:1c7b8d606e728a41ea1ccbd7264677e494e87cf630e399262ced92d4a8dac940"}, +] + +[package.dependencies] +click = ">=8.0.0" +mypy-extensions = ">=0.4.3" +packaging = ">=22.0" +pathspec = ">=0.9.0" +platformdirs = ">=2" + +[package.extras] +colorama = ["colorama (>=0.4.3)"] +d = ["aiohttp (>=3.7.4)"] +jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] +uvloop = ["uvloop (>=0.15.2)"] + +[[package]] +name = "click" +version = "8.1.3" +description = "Composable command line interface toolkit" +optional = false +python-versions = ">=3.7" +files = [ + {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, + {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "cython" +version = "0.29.35" +description = "The Cython compiler for writing C extensions for the Python language." +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "Cython-0.29.35-cp27-cp27m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fb8c11cd3e2d5ab7c2da78c5698e527ecbe469437326811562a3fbf4c5780ae4"}, + {file = "Cython-0.29.35-cp27-cp27m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:9e54b4bee55fec952333126147b89c195ebe1d60e8e492ec778916ca5ca03151"}, + {file = "Cython-0.29.35-cp27-cp27mu-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ba534e07543b44fb5ae37e56e61072ed1021b2d6ed643dbb92afa8239a04aa83"}, + {file = "Cython-0.29.35-cp27-cp27mu-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:c1d7a9ff809fa9b4a9fe04df86c9f7f574ca31c2ad896462a97ea89523db286a"}, + {file = "Cython-0.29.35-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:247d585d8e49f002e522f3420751a4b3da0cf8532ef64d382e0bc9b4c840642c"}, + {file = "Cython-0.29.35-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:ef2fc6f81aa8fb512535b01199fbe0d0ecafb8a29f261055e4b3f103c7bd6c75"}, + {file = "Cython-0.29.35-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_24_i686.whl", hash = "sha256:be7e1f98a359408186025f84d28d243e4527acb976f06b8ae8441dc5db204280"}, + {file = "Cython-0.29.35-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2e1e5d62f15ea4fa4a8bc76e4fcc2ea313a8afe70488b7b870716bcfb12b8246"}, + {file = "Cython-0.29.35-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:75541567a2de1f893d247a7f9aa300dff5662fb33822a5fb75bc9621369b8ef0"}, + {file = "Cython-0.29.35-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:99477c1d4a105a562c05d43cc01905b6711f0a6a558d90f20c7aee0fb23d59d5"}, + {file = "Cython-0.29.35-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_24_i686.whl", hash = "sha256:c44bb47b314abc743705c7d038d351ffc3a34b95ab59b04b8cb27cf781b44ae8"}, + {file = "Cython-0.29.35-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:94859c3fd90767995b33d803edecad21e73749823db468d34f21e80451a11a99"}, + {file = "Cython-0.29.35-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:5a47974f3ebccf25702ffdd569904f7807ea1ef0830987c133877fabefdc4bab"}, + {file = "Cython-0.29.35-cp35-cp35m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:520c50d1875627c111900d7184fd658e32967a3ef807dc2fbc252e384839cbcf"}, + {file = "Cython-0.29.35-cp35-cp35m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:516abc754f15b84d6a8e71c8abd90e10346ea86001563480f0be1b349d09c6b8"}, + {file = "Cython-0.29.35-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:c38e2c1e94b596132454b29757536d5afa810011d8bcb86918cc6693d2302940"}, + {file = "Cython-0.29.35-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:511f3adfb2db4db2eb882f892525db18a3a21803830474d2fa8b7a1a0f406985"}, + {file = "Cython-0.29.35-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_24_i686.whl", hash = "sha256:445e092708c26b357c97b3c68ea3eab31846fc9c1360bb150225f340c20322ec"}, + {file = "Cython-0.29.35-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3da42ef5b71674e4864b6afbe1bcacba75807684e22b6337f753cf297ae4e2d2"}, + {file = "Cython-0.29.35-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:db695a19968a54b9ac53048c723234b4f0db7409def0a5c5517237202e7a9b92"}, + {file = "Cython-0.29.35-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:156ae92bedcd8261b5259724e2dc4d8eb12ac29159359e34c8358b65d24430ac"}, + {file = "Cython-0.29.35-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:ea1c166336188630cd3e48aea4bbe06ea1bab444624e31c78973fffcae1cf708"}, + {file = "Cython-0.29.35-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:e7b1901b03c37a082ba405e2cf73a57091e835c7af35f664f9dd1d855a992ad5"}, + {file = "Cython-0.29.35-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_24_i686.whl", hash = "sha256:27f58d0dd53a8ffb614814c725d3ee3f136e53178611f7f769ff358f69e50502"}, + {file = "Cython-0.29.35-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6c19e2ba027d2e9e2d88a08aa6007344be781ed99bc0924deb237ec52ca14c09"}, + {file = "Cython-0.29.35-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:b63ea04db03190dc8b25d167598989be5c1fe9fc3121d7802c0aafc8a4ec383f"}, + {file = "Cython-0.29.35-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:5cdd65f7d85e15f1662c75d85d837c20d5c68acdd1029bfd08fb44c4422d7d9b"}, + {file = "Cython-0.29.35-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:c17c876db737e1183d18d23db9cc31a9f565c113a32523c672af72f6497e382f"}, + {file = "Cython-0.29.35-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:2a2f2fb9b1c0a4a3890713127fba55a38d2cf1619b2570c43c92a93fee80111a"}, + {file = "Cython-0.29.35-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_24_i686.whl", hash = "sha256:a1ad51612ff6cfe05cd58f584f01373d64906bb0c860a067c6441359ff10464f"}, + {file = "Cython-0.29.35-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3cd717eee52072be8244bb07f0e4126f893214d2dfd1ba8b38b533e1ffec4f8a"}, + {file = "Cython-0.29.35-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:acab11c834cbe8fb7b71f9f7b4c4655afd82ffadb1be93d5354a67702fcee69d"}, + {file = "Cython-0.29.35-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8841158f274896702afe732571d37be22868a301275f952f6280547b25280538"}, + {file = "Cython-0.29.35-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:0a9334d137bd42fca34b6b413063e19c194ba760846f34804ea1fb477cbe9a88"}, + {file = "Cython-0.29.35-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:c4cd7de707938b8385cd1f88e1446228fbfe09af7822fa13877a4374c4881198"}, + {file = "Cython-0.29.35-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_24_i686.whl", hash = "sha256:05b7ede0b0eb1c6b9bd748fa67c5ebf3c3560d04d7c8a1486183ddd099de5a00"}, + {file = "Cython-0.29.35-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:537bc1e0ed9bf7289c80f39a9a9359f5649068647631996313f77ba57afde40b"}, + {file = "Cython-0.29.35-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:402307ad6fd209816cf539680035ef79cce171288cb98f81f3f11ea8ef3afd99"}, + {file = "Cython-0.29.35-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:563a02ea675ed6321d6257df067c89f17b89a63487ef8b9ce0d598e88e7ff0bd"}, + {file = "Cython-0.29.35-py2.py3-none-any.whl", hash = "sha256:417703dc67c447089258ab4b3d217f9c03894574e4a0d6c50648a208bc8352bb"}, + {file = "Cython-0.29.35.tar.gz", hash = "sha256:6e381fa0bf08b3c26ec2f616b19ae852c06f5750f4290118bf986b6f85c8c527"}, +] + +[[package]] +name = "dill" +version = "0.3.6" +description = "serialize all of python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dill-0.3.6-py3-none-any.whl", hash = "sha256:a07ffd2351b8c678dfc4a856a3005f8067aea51d6ba6c700796a4d9e280f39f0"}, + {file = "dill-0.3.6.tar.gz", hash = "sha256:e5db55f3687856d8fbdab002ed78544e1c4559a130302693d839dfe8f93f2373"}, +] + +[package.extras] +graph = ["objgraph (>=1.7.2)"] + +[[package]] +name = "isort" +version = "5.12.0" +description = "A Python utility / library to sort Python imports." +optional = false +python-versions = ">=3.8.0" +files = [ + {file = "isort-5.12.0-py3-none-any.whl", hash = "sha256:f84c2818376e66cf843d497486ea8fed8700b340f308f076c6fb1229dff318b6"}, + {file = "isort-5.12.0.tar.gz", hash = "sha256:8bef7dde241278824a6d83f44a544709b065191b95b6e50894bdc722fcba0504"}, +] + +[package.extras] +colors = ["colorama (>=0.4.3)"] +pipfile-deprecated-finder = ["pip-shims (>=0.5.2)", "pipreqs", "requirementslib"] +plugins = ["setuptools"] +requirements-deprecated-finder = ["pip-api", "pipreqs"] + +[[package]] +name = "lazy-object-proxy" +version = "1.9.0" +description = "A fast and thorough lazy object proxy." +optional = false +python-versions = ">=3.7" +files = [ + {file = "lazy-object-proxy-1.9.0.tar.gz", hash = "sha256:659fb5809fa4629b8a1ac5106f669cfc7bef26fbb389dda53b3e010d1ac4ebae"}, + {file = "lazy_object_proxy-1.9.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b40387277b0ed2d0602b8293b94d7257e17d1479e257b4de114ea11a8cb7f2d7"}, + {file = "lazy_object_proxy-1.9.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8c6cfb338b133fbdbc5cfaa10fe3c6aeea827db80c978dbd13bc9dd8526b7d4"}, + {file = "lazy_object_proxy-1.9.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:721532711daa7db0d8b779b0bb0318fa87af1c10d7fe5e52ef30f8eff254d0cd"}, + {file = "lazy_object_proxy-1.9.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:66a3de4a3ec06cd8af3f61b8e1ec67614fbb7c995d02fa224813cb7afefee701"}, + {file = "lazy_object_proxy-1.9.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:1aa3de4088c89a1b69f8ec0dcc169aa725b0ff017899ac568fe44ddc1396df46"}, + {file = "lazy_object_proxy-1.9.0-cp310-cp310-win32.whl", hash = "sha256:f0705c376533ed2a9e5e97aacdbfe04cecd71e0aa84c7c0595d02ef93b6e4455"}, + {file = "lazy_object_proxy-1.9.0-cp310-cp310-win_amd64.whl", hash = "sha256:ea806fd4c37bf7e7ad82537b0757999264d5f70c45468447bb2b91afdbe73a6e"}, + {file = "lazy_object_proxy-1.9.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:946d27deaff6cf8452ed0dba83ba38839a87f4f7a9732e8f9fd4107b21e6ff07"}, + {file = "lazy_object_proxy-1.9.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:79a31b086e7e68b24b99b23d57723ef7e2c6d81ed21007b6281ebcd1688acb0a"}, + {file = "lazy_object_proxy-1.9.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f699ac1c768270c9e384e4cbd268d6e67aebcfae6cd623b4d7c3bfde5a35db59"}, + {file = "lazy_object_proxy-1.9.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:bfb38f9ffb53b942f2b5954e0f610f1e721ccebe9cce9025a38c8ccf4a5183a4"}, + {file = "lazy_object_proxy-1.9.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:189bbd5d41ae7a498397287c408617fe5c48633e7755287b21d741f7db2706a9"}, + {file = "lazy_object_proxy-1.9.0-cp311-cp311-win32.whl", hash = "sha256:81fc4d08b062b535d95c9ea70dbe8a335c45c04029878e62d744bdced5141586"}, + {file = "lazy_object_proxy-1.9.0-cp311-cp311-win_amd64.whl", hash = "sha256:f2457189d8257dd41ae9b434ba33298aec198e30adf2dcdaaa3a28b9994f6adb"}, + {file = "lazy_object_proxy-1.9.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d9e25ef10a39e8afe59a5c348a4dbf29b4868ab76269f81ce1674494e2565a6e"}, + {file = "lazy_object_proxy-1.9.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cbf9b082426036e19c6924a9ce90c740a9861e2bdc27a4834fd0a910742ac1e8"}, + {file = "lazy_object_proxy-1.9.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f5fa4a61ce2438267163891961cfd5e32ec97a2c444e5b842d574251ade27d2"}, + {file = "lazy_object_proxy-1.9.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:8fa02eaab317b1e9e03f69aab1f91e120e7899b392c4fc19807a8278a07a97e8"}, + {file = "lazy_object_proxy-1.9.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:e7c21c95cae3c05c14aafffe2865bbd5e377cfc1348c4f7751d9dc9a48ca4bda"}, + {file = "lazy_object_proxy-1.9.0-cp37-cp37m-win32.whl", hash = "sha256:f12ad7126ae0c98d601a7ee504c1122bcef553d1d5e0c3bfa77b16b3968d2734"}, + {file = "lazy_object_proxy-1.9.0-cp37-cp37m-win_amd64.whl", hash = "sha256:edd20c5a55acb67c7ed471fa2b5fb66cb17f61430b7a6b9c3b4a1e40293b1671"}, + {file = "lazy_object_proxy-1.9.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2d0daa332786cf3bb49e10dc6a17a52f6a8f9601b4cf5c295a4f85854d61de63"}, + {file = "lazy_object_proxy-1.9.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cd077f3d04a58e83d04b20e334f678c2b0ff9879b9375ed107d5d07ff160171"}, + {file = "lazy_object_proxy-1.9.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:660c94ea760b3ce47d1855a30984c78327500493d396eac4dfd8bd82041b22be"}, + {file = "lazy_object_proxy-1.9.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:212774e4dfa851e74d393a2370871e174d7ff0ebc980907723bb67d25c8a7c30"}, + {file = "lazy_object_proxy-1.9.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:f0117049dd1d5635bbff65444496c90e0baa48ea405125c088e93d9cf4525b11"}, + {file = "lazy_object_proxy-1.9.0-cp38-cp38-win32.whl", hash = "sha256:0a891e4e41b54fd5b8313b96399f8b0e173bbbfc03c7631f01efbe29bb0bcf82"}, + {file = "lazy_object_proxy-1.9.0-cp38-cp38-win_amd64.whl", hash = "sha256:9990d8e71b9f6488e91ad25f322898c136b008d87bf852ff65391b004da5e17b"}, + {file = "lazy_object_proxy-1.9.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9e7551208b2aded9c1447453ee366f1c4070602b3d932ace044715d89666899b"}, + {file = "lazy_object_proxy-1.9.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5f83ac4d83ef0ab017683d715ed356e30dd48a93746309c8f3517e1287523ef4"}, + {file = "lazy_object_proxy-1.9.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7322c3d6f1766d4ef1e51a465f47955f1e8123caee67dd641e67d539a534d006"}, + {file = "lazy_object_proxy-1.9.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:18b78ec83edbbeb69efdc0e9c1cb41a3b1b1ed11ddd8ded602464c3fc6020494"}, + {file = "lazy_object_proxy-1.9.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:09763491ce220c0299688940f8dc2c5d05fd1f45af1e42e636b2e8b2303e4382"}, + {file = "lazy_object_proxy-1.9.0-cp39-cp39-win32.whl", hash = "sha256:9090d8e53235aa280fc9239a86ae3ea8ac58eff66a705fa6aa2ec4968b95c821"}, + {file = "lazy_object_proxy-1.9.0-cp39-cp39-win_amd64.whl", hash = "sha256:db1c1722726f47e10e0b5fdbf15ac3b8adb58c091d12b3ab713965795036985f"}, +] + +[[package]] +name = "mccabe" +version = "0.7.0" +description = "McCabe checker, plugin for flake8" +optional = false +python-versions = ">=3.6" +files = [ + {file = "mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e"}, + {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"}, +] + +[[package]] +name = "mslex" +version = "0.3.0" +description = "shlex for windows" +optional = false +python-versions = ">=3.5" +files = [ + {file = "mslex-0.3.0-py2.py3-none-any.whl", hash = "sha256:380cb14abf8fabf40e56df5c8b21a6d533dc5cbdcfe42406bbf08dda8f42e42a"}, + {file = "mslex-0.3.0.tar.gz", hash = "sha256:4a1ac3f25025cad78ad2fe499dd16d42759f7a3801645399cce5c404415daa97"}, +] + +[[package]] +name = "mypy-extensions" +version = "1.0.0" +description = "Type system extensions for programs checked with the mypy type checker." +optional = false +python-versions = ">=3.5" +files = [ + {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, + {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, +] + +[[package]] +name = "packaging" +version = "23.1" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.1-py3-none-any.whl", hash = "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61"}, + {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, +] + +[[package]] +name = "pathspec" +version = "0.11.1" +description = "Utility library for gitignore style pattern matching of file paths." +optional = false +python-versions = ">=3.7" +files = [ + {file = "pathspec-0.11.1-py3-none-any.whl", hash = "sha256:d8af70af76652554bd134c22b3e8a1cc46ed7d91edcdd721ef1a0c51a84a5293"}, + {file = "pathspec-0.11.1.tar.gz", hash = "sha256:2798de800fa92780e33acca925945e9a19a133b715067cf165b8866c15a31687"}, +] + +[[package]] +name = "platformdirs" +version = "3.6.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.7" +files = [ + {file = "platformdirs-3.6.0-py3-none-any.whl", hash = "sha256:ffa199e3fbab8365778c4a10e1fbf1b9cd50707de826eb304b50e57ec0cc8d38"}, + {file = "platformdirs-3.6.0.tar.gz", hash = "sha256:57e28820ca8094678b807ff529196506d7a21e17156cb1cddb3e74cebce54640"}, +] + +[package.extras] +docs = ["furo (>=2023.5.20)", "proselint (>=0.13)", "sphinx (>=7.0.1)", "sphinx-autodoc-typehints (>=1.23,!=1.23.4)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.3.1)", "pytest-cov (>=4.1)", "pytest-mock (>=3.10)"] + +[[package]] +name = "psutil" +version = "5.9.5" +description = "Cross-platform lib for process and system monitoring in Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "psutil-5.9.5-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:be8929ce4313f9f8146caad4272f6abb8bf99fc6cf59344a3167ecd74f4f203f"}, + {file = "psutil-5.9.5-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:ab8ed1a1d77c95453db1ae00a3f9c50227ebd955437bcf2a574ba8adbf6a74d5"}, + {file = "psutil-5.9.5-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:4aef137f3345082a3d3232187aeb4ac4ef959ba3d7c10c33dd73763fbc063da4"}, + {file = "psutil-5.9.5-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:ea8518d152174e1249c4f2a1c89e3e6065941df2fa13a1ab45327716a23c2b48"}, + {file = "psutil-5.9.5-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:acf2aef9391710afded549ff602b5887d7a2349831ae4c26be7c807c0a39fac4"}, + {file = "psutil-5.9.5-cp27-none-win32.whl", hash = "sha256:5b9b8cb93f507e8dbaf22af6a2fd0ccbe8244bf30b1baad6b3954e935157ae3f"}, + {file = "psutil-5.9.5-cp27-none-win_amd64.whl", hash = "sha256:8c5f7c5a052d1d567db4ddd231a9d27a74e8e4a9c3f44b1032762bd7b9fdcd42"}, + {file = "psutil-5.9.5-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:3c6f686f4225553615612f6d9bc21f1c0e305f75d7d8454f9b46e901778e7217"}, + {file = "psutil-5.9.5-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7a7dd9997128a0d928ed4fb2c2d57e5102bb6089027939f3b722f3a210f9a8da"}, + {file = "psutil-5.9.5-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89518112647f1276b03ca97b65cc7f64ca587b1eb0278383017c2a0dcc26cbe4"}, + {file = "psutil-5.9.5-cp36-abi3-win32.whl", hash = "sha256:104a5cc0e31baa2bcf67900be36acde157756b9c44017b86b2c049f11957887d"}, + {file = "psutil-5.9.5-cp36-abi3-win_amd64.whl", hash = "sha256:b258c0c1c9d145a1d5ceffab1134441c4c5113b2417fafff7315a917a026c3c9"}, + {file = "psutil-5.9.5-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:c607bb3b57dc779d55e1554846352b4e358c10fff3abf3514a7a6601beebdb30"}, + {file = "psutil-5.9.5.tar.gz", hash = "sha256:5410638e4df39c54d957fc51ce03048acd8e6d60abc0f5107af51e5fb566eb3c"}, +] + +[package.extras] +test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] + +[[package]] +name = "pylint" +version = "2.17.4" +description = "python code static checker" +optional = false +python-versions = ">=3.7.2" +files = [ + {file = "pylint-2.17.4-py3-none-any.whl", hash = "sha256:7a1145fb08c251bdb5cca11739722ce64a63db479283d10ce718b2460e54123c"}, + {file = "pylint-2.17.4.tar.gz", hash = "sha256:5dcf1d9e19f41f38e4e85d10f511e5b9c35e1aa74251bf95cdd8cb23584e2db1"}, +] + +[package.dependencies] +astroid = ">=2.15.4,<=2.17.0-dev0" +colorama = {version = ">=0.4.5", markers = "sys_platform == \"win32\""} +dill = {version = ">=0.3.6", markers = "python_version >= \"3.11\""} +isort = ">=4.2.5,<6" +mccabe = ">=0.6,<0.8" +platformdirs = ">=2.2.0" +tomlkit = ">=0.10.1" + +[package.extras] +spelling = ["pyenchant (>=3.2,<4.0)"] +testutils = ["gitpython (>3)"] + +[[package]] +name = "taskipy" +version = "1.11.0" +description = "tasks runner for python projects" +optional = false +python-versions = ">=3.6,<4.0" +files = [ + {file = "taskipy-1.11.0-py3-none-any.whl", hash = "sha256:4e40cd41747a54bc8a9b3c21057c25cac645309c2d8ac897bdc1e7235e9c900e"}, + {file = "taskipy-1.11.0.tar.gz", hash = "sha256:521e8b3b65dc1ff9bb036cae989dbe5aec1626a61cf4744e5c0d0d2450c7fcb4"}, +] + +[package.dependencies] +colorama = ">=0.4.4,<0.5.0" +mslex = {version = ">=0.3.0,<0.4.0", markers = "sys_platform == \"win32\""} +psutil = ">=5.7.2,<6.0.0" +tomli = {version = ">=2.0.1,<3.0.0", markers = "python_version >= \"3.7\" and python_version < \"4.0\""} + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] + +[[package]] +name = "tomlkit" +version = "0.11.8" +description = "Style preserving TOML library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tomlkit-0.11.8-py3-none-any.whl", hash = "sha256:8c726c4c202bdb148667835f68d68780b9a003a9ec34167b6c673b38eff2a171"}, + {file = "tomlkit-0.11.8.tar.gz", hash = "sha256:9330fc7faa1db67b541b28e62018c17d20be733177d290a13b24c62d1614e0c3"}, +] + +[[package]] +name = "wrapt" +version = "1.15.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" +files = [ + {file = "wrapt-1.15.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:ca1cccf838cd28d5a0883b342474c630ac48cac5df0ee6eacc9c7290f76b11c1"}, + {file = "wrapt-1.15.0-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:e826aadda3cae59295b95343db8f3d965fb31059da7de01ee8d1c40a60398b29"}, + {file = "wrapt-1.15.0-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:5fc8e02f5984a55d2c653f5fea93531e9836abbd84342c1d1e17abc4a15084c2"}, + {file = "wrapt-1.15.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:96e25c8603a155559231c19c0349245eeb4ac0096fe3c1d0be5c47e075bd4f46"}, + {file = "wrapt-1.15.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:40737a081d7497efea35ab9304b829b857f21558acfc7b3272f908d33b0d9d4c"}, + {file = "wrapt-1.15.0-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:f87ec75864c37c4c6cb908d282e1969e79763e0d9becdfe9fe5473b7bb1e5f09"}, + {file = "wrapt-1.15.0-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:1286eb30261894e4c70d124d44b7fd07825340869945c79d05bda53a40caa079"}, + {file = "wrapt-1.15.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:493d389a2b63c88ad56cdc35d0fa5752daac56ca755805b1b0c530f785767d5e"}, + {file = "wrapt-1.15.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:58d7a75d731e8c63614222bcb21dd992b4ab01a399f1f09dd82af17bbfc2368a"}, + {file = "wrapt-1.15.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:21f6d9a0d5b3a207cdf7acf8e58d7d13d463e639f0c7e01d82cdb671e6cb7923"}, + {file = "wrapt-1.15.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ce42618f67741d4697684e501ef02f29e758a123aa2d669e2d964ff734ee00ee"}, + {file = "wrapt-1.15.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:41d07d029dd4157ae27beab04d22b8e261eddfc6ecd64ff7000b10dc8b3a5727"}, + {file = "wrapt-1.15.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:54accd4b8bc202966bafafd16e69da9d5640ff92389d33d28555c5fd4f25ccb7"}, + {file = "wrapt-1.15.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2fbfbca668dd15b744418265a9607baa970c347eefd0db6a518aaf0cfbd153c0"}, + {file = "wrapt-1.15.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:76e9c727a874b4856d11a32fb0b389afc61ce8aaf281ada613713ddeadd1cfec"}, + {file = "wrapt-1.15.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e20076a211cd6f9b44a6be58f7eeafa7ab5720eb796975d0c03f05b47d89eb90"}, + {file = "wrapt-1.15.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a74d56552ddbde46c246b5b89199cb3fd182f9c346c784e1a93e4dc3f5ec9975"}, + {file = "wrapt-1.15.0-cp310-cp310-win32.whl", hash = "sha256:26458da5653aa5b3d8dc8b24192f574a58984c749401f98fff994d41d3f08da1"}, + {file = "wrapt-1.15.0-cp310-cp310-win_amd64.whl", hash = "sha256:75760a47c06b5974aa5e01949bf7e66d2af4d08cb8c1d6516af5e39595397f5e"}, + {file = "wrapt-1.15.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ba1711cda2d30634a7e452fc79eabcadaffedf241ff206db2ee93dd2c89a60e7"}, + {file = "wrapt-1.15.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:56374914b132c702aa9aa9959c550004b8847148f95e1b824772d453ac204a72"}, + {file = "wrapt-1.15.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a89ce3fd220ff144bd9d54da333ec0de0399b52c9ac3d2ce34b569cf1a5748fb"}, + {file = "wrapt-1.15.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3bbe623731d03b186b3d6b0d6f51865bf598587c38d6f7b0be2e27414f7f214e"}, + {file = "wrapt-1.15.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3abbe948c3cbde2689370a262a8d04e32ec2dd4f27103669a45c6929bcdbfe7c"}, + {file = "wrapt-1.15.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b67b819628e3b748fd3c2192c15fb951f549d0f47c0449af0764d7647302fda3"}, + {file = "wrapt-1.15.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:7eebcdbe3677e58dd4c0e03b4f2cfa346ed4049687d839adad68cc38bb559c92"}, + {file = "wrapt-1.15.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:74934ebd71950e3db69960a7da29204f89624dde411afbfb3b4858c1409b1e98"}, + {file = "wrapt-1.15.0-cp311-cp311-win32.whl", hash = "sha256:bd84395aab8e4d36263cd1b9308cd504f6cf713b7d6d3ce25ea55670baec5416"}, + {file = "wrapt-1.15.0-cp311-cp311-win_amd64.whl", hash = "sha256:a487f72a25904e2b4bbc0817ce7a8de94363bd7e79890510174da9d901c38705"}, + {file = "wrapt-1.15.0-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:4ff0d20f2e670800d3ed2b220d40984162089a6e2c9646fdb09b85e6f9a8fc29"}, + {file = "wrapt-1.15.0-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9ed6aa0726b9b60911f4aed8ec5b8dd7bf3491476015819f56473ffaef8959bd"}, + {file = "wrapt-1.15.0-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:896689fddba4f23ef7c718279e42f8834041a21342d95e56922e1c10c0cc7afb"}, + {file = "wrapt-1.15.0-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:75669d77bb2c071333417617a235324a1618dba66f82a750362eccbe5b61d248"}, + {file = "wrapt-1.15.0-cp35-cp35m-win32.whl", hash = "sha256:fbec11614dba0424ca72f4e8ba3c420dba07b4a7c206c8c8e4e73f2e98f4c559"}, + {file = "wrapt-1.15.0-cp35-cp35m-win_amd64.whl", hash = "sha256:fd69666217b62fa5d7c6aa88e507493a34dec4fa20c5bd925e4bc12fce586639"}, + {file = "wrapt-1.15.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:b0724f05c396b0a4c36a3226c31648385deb6a65d8992644c12a4963c70326ba"}, + {file = "wrapt-1.15.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bbeccb1aa40ab88cd29e6c7d8585582c99548f55f9b2581dfc5ba68c59a85752"}, + {file = "wrapt-1.15.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:38adf7198f8f154502883242f9fe7333ab05a5b02de7d83aa2d88ea621f13364"}, + {file = "wrapt-1.15.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:578383d740457fa790fdf85e6d346fda1416a40549fe8db08e5e9bd281c6a475"}, + {file = "wrapt-1.15.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:a4cbb9ff5795cd66f0066bdf5947f170f5d63a9274f99bdbca02fd973adcf2a8"}, + {file = "wrapt-1.15.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:af5bd9ccb188f6a5fdda9f1f09d9f4c86cc8a539bd48a0bfdc97723970348418"}, + {file = "wrapt-1.15.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:b56d5519e470d3f2fe4aa7585f0632b060d532d0696c5bdfb5e8319e1d0f69a2"}, + {file = "wrapt-1.15.0-cp36-cp36m-win32.whl", hash = "sha256:77d4c1b881076c3ba173484dfa53d3582c1c8ff1f914c6461ab70c8428b796c1"}, + {file = "wrapt-1.15.0-cp36-cp36m-win_amd64.whl", hash = "sha256:077ff0d1f9d9e4ce6476c1a924a3332452c1406e59d90a2cf24aeb29eeac9420"}, + {file = "wrapt-1.15.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5c5aa28df055697d7c37d2099a7bc09f559d5053c3349b1ad0c39000e611d317"}, + {file = "wrapt-1.15.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3a8564f283394634a7a7054b7983e47dbf39c07712d7b177b37e03f2467a024e"}, + {file = "wrapt-1.15.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:780c82a41dc493b62fc5884fb1d3a3b81106642c5c5c78d6a0d4cbe96d62ba7e"}, + {file = "wrapt-1.15.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e169e957c33576f47e21864cf3fc9ff47c223a4ebca8960079b8bd36cb014fd0"}, + {file = "wrapt-1.15.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:b02f21c1e2074943312d03d243ac4388319f2456576b2c6023041c4d57cd7019"}, + {file = "wrapt-1.15.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:f2e69b3ed24544b0d3dbe2c5c0ba5153ce50dcebb576fdc4696d52aa22db6034"}, + {file = "wrapt-1.15.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d787272ed958a05b2c86311d3a4135d3c2aeea4fc655705f074130aa57d71653"}, + {file = "wrapt-1.15.0-cp37-cp37m-win32.whl", hash = "sha256:02fce1852f755f44f95af51f69d22e45080102e9d00258053b79367d07af39c0"}, + {file = "wrapt-1.15.0-cp37-cp37m-win_amd64.whl", hash = "sha256:abd52a09d03adf9c763d706df707c343293d5d106aea53483e0ec8d9e310ad5e"}, + {file = "wrapt-1.15.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:cdb4f085756c96a3af04e6eca7f08b1345e94b53af8921b25c72f096e704e145"}, + {file = "wrapt-1.15.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:230ae493696a371f1dbffaad3dafbb742a4d27a0afd2b1aecebe52b740167e7f"}, + {file = "wrapt-1.15.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63424c681923b9f3bfbc5e3205aafe790904053d42ddcc08542181a30a7a51bd"}, + {file = "wrapt-1.15.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d6bcbfc99f55655c3d93feb7ef3800bd5bbe963a755687cbf1f490a71fb7794b"}, + {file = "wrapt-1.15.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c99f4309f5145b93eca6e35ac1a988f0dc0a7ccf9ccdcd78d3c0adf57224e62f"}, + {file = "wrapt-1.15.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b130fe77361d6771ecf5a219d8e0817d61b236b7d8b37cc045172e574ed219e6"}, + {file = "wrapt-1.15.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:96177eb5645b1c6985f5c11d03fc2dbda9ad24ec0f3a46dcce91445747e15094"}, + {file = "wrapt-1.15.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5fe3e099cf07d0fb5a1e23d399e5d4d1ca3e6dfcbe5c8570ccff3e9208274f7"}, + {file = "wrapt-1.15.0-cp38-cp38-win32.whl", hash = "sha256:abd8f36c99512755b8456047b7be10372fca271bf1467a1caa88db991e7c421b"}, + {file = "wrapt-1.15.0-cp38-cp38-win_amd64.whl", hash = "sha256:b06fa97478a5f478fb05e1980980a7cdf2712015493b44d0c87606c1513ed5b1"}, + {file = "wrapt-1.15.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2e51de54d4fb8fb50d6ee8327f9828306a959ae394d3e01a1ba8b2f937747d86"}, + {file = "wrapt-1.15.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0970ddb69bba00670e58955f8019bec4a42d1785db3faa043c33d81de2bf843c"}, + {file = "wrapt-1.15.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76407ab327158c510f44ded207e2f76b657303e17cb7a572ffe2f5a8a48aa04d"}, + {file = "wrapt-1.15.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cd525e0e52a5ff16653a3fc9e3dd827981917d34996600bbc34c05d048ca35cc"}, + {file = "wrapt-1.15.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d37ac69edc5614b90516807de32d08cb8e7b12260a285ee330955604ed9dd29"}, + {file = "wrapt-1.15.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:078e2a1a86544e644a68422f881c48b84fef6d18f8c7a957ffd3f2e0a74a0d4a"}, + {file = "wrapt-1.15.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:2cf56d0e237280baed46f0b5316661da892565ff58309d4d2ed7dba763d984b8"}, + {file = "wrapt-1.15.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7dc0713bf81287a00516ef43137273b23ee414fe41a3c14be10dd95ed98a2df9"}, + {file = "wrapt-1.15.0-cp39-cp39-win32.whl", hash = "sha256:46ed616d5fb42f98630ed70c3529541408166c22cdfd4540b88d5f21006b0eff"}, + {file = "wrapt-1.15.0-cp39-cp39-win_amd64.whl", hash = "sha256:eef4d64c650f33347c1f9266fa5ae001440b232ad9b98f1f43dfe7a79435c0a6"}, + {file = "wrapt-1.15.0-py3-none-any.whl", hash = "sha256:64b1df0f83706b4ef4cfb4fb0e4c2669100fd7ecacfb59e091fad300d4e04640"}, + {file = "wrapt-1.15.0.tar.gz", hash = "sha256:d06730c6aed78cee4126234cf2d071e01b44b915e725a6cb439a879ec9754a3a"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "==3.11.3" +content-hash = "6f52914e88663d00904af8bd59a8246699dfa700d50edff9057b6dc82eabe897" diff --git a/pyproject.toml b/pyproject.toml index 2fa01dd..e5d5608 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,10 +14,14 @@ bjoern = "^3.2.2" black = "^23.3.0" pylint = "^2.17.4" taskipy = "^1.11.0" +cython = "^0.29.35" [tool.taskipy.tasks] lint = { cmd = "pylint microwrap", help = "check code style with pylint" } +pre_compile = { cmd = "cython -3 --embed -o microwrap.c microwrap/microwrap.py", help = "transpile code to C with cython" } +compile = { cmd = "gcc -I<> -L<> -lpython3.11 -o microwrap microwrap.c ", help = "compile code to single executable with cython" } + [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" From ee1e41d904ef6a8b29cbbb5636bffe4e3e0d469d Mon Sep 17 00:00:00 2001 From: Saejin Mahlau-Heinert Date: Tue, 20 Jun 2023 16:44:35 -0500 Subject: [PATCH 03/32] Update README --- README.md | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 1924447..83a6f9a 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,35 @@ # MicroWrap -MicroWrap is a base container image designed to streamline and simplify the process of developing and deploying microservices through the use of containerization. +MicroWrap is a base container image designed to streamline and simplify the process of developing and deploying microservices through the use of containerization. Using it is as simple as writing a Dockerfile for your application, and including a `microwrap.json` configuration; see the example below: + +```Dockerfile +# Create an image using microwrap as the base to serve as our runtime image +FROM michionlion/microwrap:latest +# Configure microwrap +COPY microwrap.json /microwrap.json +# Upload executable to expose as a service +COPY version.sh /version.sh +``` + +MicroWrap functions as an executable wrapper, abstracting the complexities of network communication and service execution away from the application itself. It consists of an HTTP server that listens for incoming HTTP requests and translates those HTTP requests to command-line invocations of the wrapped executable. The translation process supports parameters -- URL parameters embedded in the request will become `--option value` strings passed to the wrapped executable. The standard output of the wrapped executable will be returned as the body of the response to the triggering request. -MicroWrap functions as an executable wrapper, abstracting the complexities of network communication and service execution away from the application itself. It consists of an HTTP server that listens for incoming HTTP requests and translates those HTTP requests to command-line invocations of the wrapped executable. The translation process supports parameters -- URL parameters embedded in the request will become `--option value` strings passed to the wrapped executable. The standard output of the wrapped executable will be returned as the body of the response to the triggering request. For example, the request `GET http://localhost/execute?option1=test2&flag1` would trigger the invocation `/executable/path --option1 "test2" --flag1`, and the standard output would be returned as the body of the response to the `GET` HTTP request. +As an example, suppose the following request was made to a container running microwrap: + +```shell +http GET http://$HOST:$PORT/start?option1=test2&flag1 +``` + +This request would trigger microwrap to execute its configured executable with: + +```shell +/executable/path --option1 "test2" --flag1` +``` -The base container image MicroWrap defines should be used as the base image for a further Dockerfile build, which can specify mounting locations, compile or upload the executable to be wrapped, and configure MicroWrap. An example container image using MicroWrap to run a version service is defined in this repository at `example/Dockerfile`. +The standard output of the execution would be returned as the body of the response to the `GET` HTTP request, and if the executable exits with a non-zero return code, an HTTP 500 Internal Server Error is returned (with the body being the concatenated standard output and standard error streams). ## Usage -To make your application a containerized service, you will need to write a Dockerfile that builds an image. This image can then be used in many different containerized environments, such as Docker, OpenShift, Kubernetes, and others. The Dockerfile for your application needs to accomplish two tasks: allow execution of your program, and configure MicroWrap. To allow your program to execute, the Dockerfile should install dependencies that your program needs, compile your program, and configure the runtime environment so that your program can execute. Additionally, you may want to prepare mount points for any folders that may need to be accessed by your program for external reading/writing, in the case that such input/output is needed. +To make your application a containerized service, you will need to write a Dockerfile that builds an image. This image can then be used in many different containerized environments, such as Docker, OpenShift, Kubernetes, and others. The Dockerfile for your application needs to accomplish two tasks: allow execution of your program, and configure MicroWrap. To allow your program to execute, the Dockerfile should install dependencies that your program needs, compile your program, and configure the runtime environment so that your program can execute. Additionally, you may want to prepare mount points for any folders that may need to be accessed by your program for external reading/writing, in the case that such input/output is needed. An example (which specifically uses a Java program, but is applicable to many different languages and technologies) is given in the `example/` directory of this repository. ## Configuration From bef6880e3633ba5efe1c87eb07bdcd794573e4c5 Mon Sep 17 00:00:00 2001 From: Saejin Mahlau-Heinert Date: Tue, 20 Jun 2023 16:49:34 -0500 Subject: [PATCH 04/32] Update example/ to support README example --- example/Dockerfile | 16 ++++++++++------ example/version.sh | 2 +- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/example/Dockerfile b/example/Dockerfile index 38e204e..f43051c 100644 --- a/example/Dockerfile +++ b/example/Dockerfile @@ -1,15 +1,19 @@ # Build the service executable in a "development" container -FROM alpine:3.18 as build - +FROM alpine:latest as build +# Add Java 11 JDK RUN apk add --no-cache openjdk11 +# Copy source code to container COPY Version.java /Version.java +# Compile source code RUN javac Version.java -# Create a new image with microwrap as the base to serve as our runtime image -FROM michionlion/microwrap:0.1 -# Install runtime dependency +# Create an image using microwrap as the base to serve as our runtime image +FROM michionlion/microwrap:latest +# Install Java 11 JRE RUN apk add --no-cache openjre11 # Configure microwrap COPY microwrap.json /microwrap.json -# Copy the executable from the build image +# Copy executable (a script to run Version.class) +COPY version.sh /version.sh +# Copy compiled code from the build image COPY --from=build /Version.class /Version.class diff --git a/example/version.sh b/example/version.sh index 2ccc254..e314e36 100755 --- a/example/version.sh +++ b/example/version.sh @@ -2,7 +2,7 @@ USE_JAVA=true -if [ "$USE_JAVA" == "true" ]; then +if [ "$USE_JAVA" == "true" && -f Version.class ]; then java Version.class else VERSION="1.0.0" From 3e232ef58febbb8fc4a8155cbf052b0868f24ff2 Mon Sep 17 00:00:00 2001 From: Saejin Mahlau-Heinert Date: Tue, 20 Jun 2023 18:34:03 -0500 Subject: [PATCH 05/32] Add minor parameter handling change and future work to README --- README.md | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 83a6f9a..c569d65 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # MicroWrap -MicroWrap is a base container image designed to streamline and simplify the process of developing and deploying microservices through the use of containerization. Using it is as simple as writing a Dockerfile for your application, and including a `microwrap.json` configuration; see the example below: +MicroWrap is a base container image designed to streamline and simplify the process of developing and deploying microservices through the use of containerization. Using it is as simple as writing a Dockerfile for your application, and including a `microwrap.json` configuration; see the example below. ```Dockerfile # Create an image using microwrap as the base to serve as our runtime image @@ -13,16 +13,16 @@ COPY version.sh /version.sh MicroWrap functions as an executable wrapper, abstracting the complexities of network communication and service execution away from the application itself. It consists of an HTTP server that listens for incoming HTTP requests and translates those HTTP requests to command-line invocations of the wrapped executable. The translation process supports parameters -- URL parameters embedded in the request will become `--option value` strings passed to the wrapped executable. The standard output of the wrapped executable will be returned as the body of the response to the triggering request. -As an example, suppose the following request was made to a container running microwrap: +As an example, suppose the following request was made to a container running microwrap. ```shell http GET http://$HOST:$PORT/start?option1=test2&flag1 ``` -This request would trigger microwrap to execute its configured executable with: +This request would trigger microwrap to execute its configured executable as follows. ```shell -/executable/path --option1 "test2" --flag1` +/executable/path --option1 "test2" --flag1 ``` The standard output of the execution would be returned as the body of the response to the `GET` HTTP request, and if the executable exits with a non-zero return code, an HTTP 500 Internal Server Error is returned (with the body being the concatenated standard output and standard error streams). @@ -36,7 +36,7 @@ To make your application a containerized service, you will need to write a Docke 1. **Executable Path** This is the location of the executable file that will be executed per request. It should be an executable file in your image. 2. **Max Active Requests** This is the number of wrapped-executable invocations to allow at one time; any requests beyond this number will be queued for future invocation. Specify `-1` for no limit. 3. **Allowed Parameters** This is a list of URL parameters that will be passed through as command-line options to the wrapped executable. Any other parameters will be ignored. -4. **Default Parameters** This is an object which is mapped to `--attribute value` strings passed to the wrapped executable that can be overridden by URL parameters. Values that are `true` and `false` will not map to `"true"` or `"false"`, but instead value-less `--flag` and `--no-flag` (for an attribute named `flag`) strings; values that are `null` or the empty string `""` will cause the parameter to be ignored. +4. **Default Parameters** This is an object which is mapped to `--attribute value` strings passed to the wrapped executable that can be overridden by URL parameters. Values that are `true` will not map to `"true"`, but instead a value-less `--flag` (for an attribute named `flag`) string; values that are `null`, `false`, or the empty string `""` will cause the parameter to be ignored. These configuration parameters should be specified in the `/microwrap.json` configuration file in your image: @@ -52,3 +52,15 @@ These configuration parameters should be specified in the `/microwrap.json` conf } } ``` + +## Future Work + +- Named invocations and status checking + - Requires specific endpoints (no more "any endpoint -> invocation"). + - `http://$HOST:$PORT/start/name?` Start an invocation named `name`; supports parameters (for giving to executable). + - `http://$HOST:$PORT/stop/name` Stop a running invocation named `name`. + - `http://$HOST:$PORT/running/name` Check if an invocation named `name` is running. + - `http://$HOST:$PORT/running` Get list of all running invocations. +- Progress reporting + - Need to have standard progress reporting by wrapped executable. + - Maybe a json file that the executable writes whenever and the `/progress` endpoint returns the contents of that file when called? From 3c247e9c40d06d8559ef18b0c89047bd3529a9ea Mon Sep 17 00:00:00 2001 From: Saejin Mahlau-Heinert Date: Tue, 20 Jun 2023 18:34:44 -0500 Subject: [PATCH 06/32] Add cython compilation scripts --- .gitignore | 4 ++++ pyproject.toml | 14 +++++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..65cc7fb --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +# Compilation temporary products +build/ +*.c +*.so diff --git a/pyproject.toml b/pyproject.toml index e5d5608..3c6ea0e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,11 +16,19 @@ pylint = "^2.17.4" taskipy = "^1.11.0" cython = "^0.29.35" +[tool.taskipy.settings] +use_vars = true + +[tool.taskipy.variables] +get_includes = "python -c \"import sysconfig; print(sysconfig.get_path('include'))\"" +get_libdir = "python -c \"import sysconfig; print(sysconfig.get_config_var('LIBDIR'))\"" +get_lib = "python -c \"import sysconfig; print(sysconfig.get_config_var('LIBRARY').replace('lib', '', 1).rstrip('.a'))\"" +get_ldflags = "python -c \"import sysconfig; print(sysconfig.get_config_var('LDFLAGS'))\"" + [tool.taskipy.tasks] lint = { cmd = "pylint microwrap", help = "check code style with pylint" } - -pre_compile = { cmd = "cython -3 --embed -o microwrap.c microwrap/microwrap.py", help = "transpile code to C with cython" } -compile = { cmd = "gcc -I<> -L<> -lpython3.11 -o microwrap microwrap.c ", help = "compile code to single executable with cython" } +pre_compile = { cmd = "mkdir -p build && cython -3 --embed -o build/microwrap.c microwrap/microwrap.py", help = "transpile code to C with cython" } +compile = { cmd = "gcc -v -I$({get_includes}) -L$({get_libdir}) -l$({get_lib}) $({get_ldflags}) -o build/microwrap build/microwrap.c", help = "compile code to single executable with cython" } [build-system] requires = ["poetry-core"] From c513e7216742c2b5e2c2b6e4bb9f3ba026808058 Mon Sep 17 00:00:00 2001 From: Saejin Mahlau-Heinert Date: Tue, 20 Jun 2023 19:57:56 -0500 Subject: [PATCH 07/32] Update documentation for maxActiveRequests such that 0 == unlimited --- README.md | 2 +- example/microwrap.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c569d65..f02c87a 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ To make your application a containerized service, you will need to write a Docke ## Configuration 1. **Executable Path** This is the location of the executable file that will be executed per request. It should be an executable file in your image. -2. **Max Active Requests** This is the number of wrapped-executable invocations to allow at one time; any requests beyond this number will be queued for future invocation. Specify `-1` for no limit. +2. **Max Active Requests** This is the number of wrapped-executable invocations to allow at one time; any requests beyond this number will be queued for future invocation. Specify `0` for no limit. 3. **Allowed Parameters** This is a list of URL parameters that will be passed through as command-line options to the wrapped executable. Any other parameters will be ignored. 4. **Default Parameters** This is an object which is mapped to `--attribute value` strings passed to the wrapped executable that can be overridden by URL parameters. Values that are `true` will not map to `"true"`, but instead a value-less `--flag` (for an attribute named `flag`) string; values that are `null`, `false`, or the empty string `""` will cause the parameter to be ignored. diff --git a/example/microwrap.json b/example/microwrap.json index 825c39a..dee7953 100644 --- a/example/microwrap.json +++ b/example/microwrap.json @@ -1,6 +1,6 @@ { "executablePath": "/version.sh", - "maxActiveRequests": -1, + "maxActiveRequests": 0, "allowedParameters": [ "include-build" ], From 238d17c0a7eaa41a978bb1b7c5b8e743db654979 Mon Sep 17 00:00:00 2001 From: Saejin Mahlau-Heinert Date: Tue, 20 Jun 2023 19:58:35 -0500 Subject: [PATCH 08/32] Add basic complete implementation --- microwrap/microwrap.py | 175 ++++++++++++++++++++++++++++------------- 1 file changed, 122 insertions(+), 53 deletions(-) diff --git a/microwrap/microwrap.py b/microwrap/microwrap.py index 1e91892..ec3cafe 100644 --- a/microwrap/microwrap.py +++ b/microwrap/microwrap.py @@ -1,75 +1,144 @@ -"""Main MicroWrap module.""" +"""MicroWrap.""" import sys import json from urllib.parse import parse_qs -import time +import wsgiref.simple_server as server +import threading +import subprocess -from typing import Any, Iterable +from typing import Any, Iterable, Dict, List, Union from wsgiref.types import WSGIEnvironment, StartResponse -# this can be replaced with any WSGI server -import bjoern as wsgi_server - USAGE_HELP = "Usage: microwrap " CONFIG_PATH = "/microwrap.json" -RESPONSE_HEADERS = [] - - -def load_config() -> dict[str, Any]: - """Load configuration from disk.""" - with open(CONFIG_PATH, encoding="utf-8") as file: - return json.load(file) - - -def execute_command(config: dict[str, Any], params: dict[str, list[str]]): - """Build the command to execute.""" - time.sleep(2) - - executable = config.get("executablePath", "") - options = config.get("defaultParameters", {}) - for key, value in params.items(): - if config.get("allowedParameters").contains(key): - options[key] = value - - arguments = [] - - for key, value in options.items(): - if value == False or value == True or value == "": - arguments.append(f"--{key}") - else: - arguments.append(f"--{key} {value}") - - # execute command - - # return output - return executable + " " + " ".join(arguments) +WORKING_DIRECTORY = "/" + + +class Config: + """A MicroWrap configuration file.""" + + def __init__(self, path: str): + with open(path, encoding="utf-8") as file: + json_config = json.load(file) + if isinstance(json_config, dict): + self.config = json_config + else: + raise ValueError("Invalid configuration file!") + + def get_allowed_params(self) -> List[str]: + """Return the list of allowed query-string parameters.""" + # TODO: verify valid config + # should be a list of strings + return self.config.get("allowedParameters", []) + + def get_default_params(self) -> Dict[str, Union[str, bool]]: + """Return the default query-string parameter values.""" + # TODO: verify valid config + # valid if (isinstance(value, str) or isinstance(value, bool)) and value != "" + # if a value is "", error should include "should be `true` to indicate a value-less option" + # if a value is not a str or bool, error should include "should be a string or boolean" + return self.config.get("defaultParameters", {}) + + def get_executable_path(self) -> str: + """Return the path to the executable.""" + # TODO: verify valid config + # should be an existing path + return self.config.get("executablePath") + + def get_max_active_requests(self) -> int: + """Return the maximum number of active requests.""" + # TODO: verify valid config + # should be a positive integer or 0 (unlimited) + return self.config.get("maxActiveRequests", 1) + + +def parse_query_params(config: Config, query_str) -> Dict[str, str]: + """Parse parameters from the request.""" + query_params = parse_qs(query_str, keep_blank_values=True) + allowed_params = config.get_allowed_params() + default_params = config.get_default_params() + + params = { + key: "" if value is True else value + for key, value in default_params + if value is not False + } + + for allowed_param in allowed_params: + value = query_params.get(allowed_param, None) + if value is not None: + params[allowed_param] = value[-1] + + return params + + +class InvocationRequest: + """An invocation request.""" + + def __init__(self, config: Dict[str, Any], env: WSGIEnvironment): + self.config = config + # self.input_body = env["wsgi.input"].read() + # self.errors = env["wsgi.errors"] + self.method = env.get("REQUEST_METHOD", "") + self.path = env.get("SCRIPT_NAME", "") + env.get("PATH_TRANSLATED", "") + self.params = parse_query_params(config, env.get("QUERY_STRING", "")) + self.arguments = None + + def get_label(self) -> str: + """Get logging prefix for this request.""" + return f"[{self.method}][{self.path}][{threading.get_native_id()}]" + + def get_arguments(self): + """Get the arguments to pass to the executable.""" + if self.arguments is None: + self.arguments = [] + for key, value in self.params: + self.arguments.append(f"--{key}") + if value.strip() != "": + self.arguments.append(value) + return self.arguments + + def execute(self) -> str: + """Execute the invocation indicated by this request.""" + print(f"{self.get_label()} Executing with arguments: {self.get_arguments()}") + proc = subprocess.run( + executable=self.config.get_executable_path(), + args=self.get_arguments(), + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=True, + text=True, + cwd=WORKING_DIRECTORY, + # start_new_session=True, + ) + return proc.stdout def microwrap(env: WSGIEnvironment, start_response: StartResponse) -> Iterable[bytes]: """WSGI application that translates HTTP requests to invocations of an arbitrary executable.""" - # input_body = environ["wsgi.input"].read() - # errors = env["wsgi.errors"] - config = load_config() - method = env.get("REQUEST_METHOD", "") - path = env.get("SCRIPT_NAME", "") + env.get("PATH_TRANSLATED", "") - params = parse_qs(env.get("QUERY_STRING", ""), keep_blank_values=True) - - print(f"[{method}][{path}] Configuration: {config}\nParameters: {params}") + try: + config = Config(CONFIG_PATH) + handler = InvocationRequest(config, env) - response_body = execute_command(config, params) - status = "200 OK" - start_response(status, RESPONSE_HEADERS) - return [response_body.encode()] + print(f"{handler.get_label()} Configuration: {config}") + print(f"{handler.get_label()} Parameters: {handler.params}") + response_body = handler.execute() + start_response("200 OK", []) + return [response_body.encode()] + except Exception as ex: + start_response("500 Internal Server Error", []) + return [str(ex).encode()] def run(host="localhost", port=80): """Run the WSGI application.""" - print(f"Listening on {host}:{port}...") - wsgi_server.run(microwrap, host, port) + print(f"Listening on http://{host}:{port}") + # Can be replaced with any WSGI server (but needs to be included by cython) + server.make_server(host, port, microwrap).serve_forever() if __name__ == "__main__": - if len(sys.argv == 1): + if len(sys.argv) == 1: run() elif len(sys.argv) == 2: if sys.argv[1] == "--help" or sys.argv[1] == "-h": @@ -77,6 +146,6 @@ def run(host="localhost", port=80): else: run(sys.argv[1]) elif len(sys.argv) == 3: - run(sys.argv[1], sys.argv[2]) + run(sys.argv[1], int(sys.argv[2])) else: print(USAGE_HELP) From 3ebda74f6ca4a4718e5a089c780296fac6245fa8 Mon Sep 17 00:00:00 2001 From: Saejin Mahlau-Heinert Date: Tue, 20 Jun 2023 20:53:35 -0500 Subject: [PATCH 09/32] Fix minor issues with initial implementation It now works locally --- microwrap/microwrap.py | 67 +++++++++++++++++++++++++++++++----------- 1 file changed, 50 insertions(+), 17 deletions(-) diff --git a/microwrap/microwrap.py b/microwrap/microwrap.py index ec3cafe..d7cf214 100644 --- a/microwrap/microwrap.py +++ b/microwrap/microwrap.py @@ -1,18 +1,16 @@ """MicroWrap.""" +import os import sys import json from urllib.parse import parse_qs import wsgiref.simple_server as server import threading import subprocess +import traceback from typing import Any, Iterable, Dict, List, Union from wsgiref.types import WSGIEnvironment, StartResponse -USAGE_HELP = "Usage: microwrap " -CONFIG_PATH = "/microwrap.json" -WORKING_DIRECTORY = "/" - class Config: """A MicroWrap configuration file.""" @@ -51,6 +49,26 @@ def get_max_active_requests(self) -> int: # should be a positive integer or 0 (unlimited) return self.config.get("maxActiveRequests", 1) + def __str__(self): + return str(self.config) + + +### +# Constants +### + + +USAGE_HELP = "Usage: microwrap " +CONFIG_PATH = "microwrap.json" +RESPONSE_HEADERS = [ + ("Server", f"MicroWrap/1.0.0 {Config(CONFIG_PATH).get_executable_path()}") +] + + +### +# Utility functions +### + def parse_query_params(config: Config, query_str) -> Dict[str, str]: """Parse parameters from the request.""" @@ -60,7 +78,7 @@ def parse_query_params(config: Config, query_str) -> Dict[str, str]: params = { key: "" if value is True else value - for key, value in default_params + for key, value in default_params.items() if value is not False } @@ -72,6 +90,11 @@ def parse_query_params(config: Config, query_str) -> Dict[str, str]: return params +### +# Request handler +### + + class InvocationRequest: """An invocation request.""" @@ -80,53 +103,63 @@ def __init__(self, config: Dict[str, Any], env: WSGIEnvironment): # self.input_body = env["wsgi.input"].read() # self.errors = env["wsgi.errors"] self.method = env.get("REQUEST_METHOD", "") - self.path = env.get("SCRIPT_NAME", "") + env.get("PATH_TRANSLATED", "") - self.params = parse_query_params(config, env.get("QUERY_STRING", "")) + self.path = env.get("PATH_INFO", "/") + self.query = env.get("QUERY_STRING", "") + self.params = parse_query_params(config, self.query) self.arguments = None def get_label(self) -> str: """Get logging prefix for this request.""" - return f"[{self.method}][{self.path}][{threading.get_native_id()}]" + return ( + f"[{threading.get_native_id()}][{self.method}][{self.path}][{self.query}]" + ) def get_arguments(self): """Get the arguments to pass to the executable.""" if self.arguments is None: self.arguments = [] - for key, value in self.params: + for key, value in self.params.items(): self.arguments.append(f"--{key}") if value.strip() != "": self.arguments.append(value) return self.arguments def execute(self) -> str: - """Execute the invocation indicated by this request.""" - print(f"{self.get_label()} Executing with arguments: {self.get_arguments()}") + """Execute the invocation requested.""" + executable = os.path.abspath(self.config.get_executable_path()) + arguments = self.get_arguments() + print(f"{self.get_label()} Executing '{executable}' with args: {arguments}") proc = subprocess.run( - executable=self.config.get_executable_path(), - args=self.get_arguments(), + executable=executable, + args=arguments, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=True, text=True, - cwd=WORKING_DIRECTORY, + cwd=".", # start_new_session=True, ) return proc.stdout +### +# Main application +### + + def microwrap(env: WSGIEnvironment, start_response: StartResponse) -> Iterable[bytes]: """WSGI application that translates HTTP requests to invocations of an arbitrary executable.""" try: config = Config(CONFIG_PATH) handler = InvocationRequest(config, env) - print(f"{handler.get_label()} Configuration: {config}") print(f"{handler.get_label()} Parameters: {handler.params}") response_body = handler.execute() - start_response("200 OK", []) + start_response("200 OK", RESPONSE_HEADERS) return [response_body.encode()] except Exception as ex: - start_response("500 Internal Server Error", []) + traceback.print_exception(ex) + start_response("500 Internal Server Error", RESPONSE_HEADERS) return [str(ex).encode()] From abd9d9e6c7bd17fc05412513d3ff6ee7e94d7d59 Mon Sep 17 00:00:00 2001 From: Saejin Mahlau-Heinert Date: Tue, 20 Jun 2023 23:10:24 -0500 Subject: [PATCH 10/32] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f02c87a..9a226ed 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ These configuration parameters should be specified in the `/microwrap.json` conf - Named invocations and status checking - Requires specific endpoints (no more "any endpoint -> invocation"). - - `http://$HOST:$PORT/start/name?` Start an invocation named `name`; supports parameters (for giving to executable). + - `http://$HOST:$PORT/start/name?` Start an invocation named `name`; supports invocation parameters. - `http://$HOST:$PORT/stop/name` Stop a running invocation named `name`. - `http://$HOST:$PORT/running/name` Check if an invocation named `name` is running. - `http://$HOST:$PORT/running` Get list of all running invocations. From 9938e152944561bd083c5ba3cd975b4da9d4f751 Mon Sep 17 00:00:00 2001 From: Saejin Mahlau-Heinert Date: Wed, 21 Jun 2023 16:12:00 -0500 Subject: [PATCH 11/32] First attempt at custom multi-threaded WSGI server --- microwrap/microwrap.py | 100 +++++++++++++++++++++++++++++++++++------ 1 file changed, 86 insertions(+), 14 deletions(-) diff --git a/microwrap/microwrap.py b/microwrap/microwrap.py index d7cf214..b2865b4 100644 --- a/microwrap/microwrap.py +++ b/microwrap/microwrap.py @@ -1,15 +1,18 @@ """MicroWrap.""" +import json import os +import signal +import subprocess import sys -import json -from urllib.parse import parse_qs -import wsgiref.simple_server as server import threading -import subprocess import traceback - -from typing import Any, Iterable, Dict, List, Union -from wsgiref.types import WSGIEnvironment, StartResponse +import wsgiref.simple_server as server +from concurrent.futures import ThreadPoolExecutor +from http.server import BaseHTTPRequestHandler +from queue import Empty, Queue +from typing import Any, Dict, Iterable, List, Union +from urllib.parse import parse_qs +from wsgiref.types import StartResponse, WSGIEnvironment class Config: @@ -60,6 +63,7 @@ def __str__(self): USAGE_HELP = "Usage: microwrap " CONFIG_PATH = "microwrap.json" +MAX_THREADS = os.cpu_count() * 4 RESPONSE_HEADERS = [ ("Server", f"MicroWrap/1.0.0 {Config(CONFIG_PATH).get_executable_path()}") ] @@ -110,9 +114,7 @@ def __init__(self, config: Dict[str, Any], env: WSGIEnvironment): def get_label(self) -> str: """Get logging prefix for this request.""" - return ( - f"[{threading.get_native_id()}][{self.method}][{self.path}][{self.query}]" - ) + return f"[{threading.current_thread().name}][{self.method}][{self.path}][{self.query}]" def get_arguments(self): """Get the arguments to pass to the executable.""" @@ -137,11 +139,75 @@ def execute(self) -> str: check=True, text=True, cwd=".", - # start_new_session=True, + start_new_session=True, ) return proc.stdout +### +# Threaded WSGI server +### + + +class ThreadedWSGIServer(server.WSGIServer): + """Threaded WSGI Server that uses a ThreadPoolExecutor to handle requests.""" + + def __init__( + self, + host: str, + port: int, + handler: BaseHTTPRequestHandler, + max_workers=None, + ): + super().__init__((host, port), handler, True) + self.requests = Queue() + self.executor = ThreadPoolExecutor(max_workers if max_workers else MAX_THREADS) + self.is_listening = False + + def process_request(self, request, client_address): + """Add requests to the queue instead of handling immediately like super does.""" + self.requests.put((request, client_address)) + print(f"Added request {request} from {client_address} to queue") + print(f"{self.requests.qsize()} requests in queue") + + def serve_forever(self, poll_interval=0.5): + """Continuously handle requests from the queue in separate threads.""" + signal.signal(signal.SIGINT, lambda _s, _f: self.shutdown()) + signal.signal(signal.SIGTERM, lambda _s, _f: self.shutdown()) + self.is_listening = True + print("Starting server...") + while self.is_listening: + try: + print(f"Pulling from queue: {self.requests.queue}") + request, client_address = self.requests.get( + block=True, timeout=poll_interval + ) + print("Retrieved request") + self.executor.submit( + self.process_request_thread, request, client_address + ) + print("Submitted request") + except Empty: + continue + + def process_request_thread(self, request, client_address): + """This function will be executed in a new thread and handle the request.""" + try: + print(f"Processing request {request} from {client_address}") + self.finish_request(request, client_address) + self.shutdown_request(request) + except Exception: + self.handle_error(request, client_address) + self.shutdown_request(request) + + def shutdown(self): + """Stop the server but wait for current requests to complete before exiting.""" + print("Waiting for requests to complete...") + self.is_listening = False + self.executor.shutdown(wait=True, cancel_futures=True) + print("Requests complete. Shutting down...") + + ### # Main application ### @@ -165,9 +231,15 @@ def microwrap(env: WSGIEnvironment, start_response: StartResponse) -> Iterable[b def run(host="localhost", port=80): """Run the WSGI application.""" - print(f"Listening on http://{host}:{port}") - # Can be replaced with any WSGI server (but needs to be included by cython) - server.make_server(host, port, microwrap).serve_forever() + print(f"Listening on http://{host}:{port}/") + httpd = ThreadedWSGIServer( + host, + port, + server.WSGIRequestHandler, + max_workers=Config(CONFIG_PATH).get_max_active_requests(), + ) + httpd.set_app(microwrap) + httpd.serve_forever(poll_interval=0.5) if __name__ == "__main__": From b91dcdaef3120b1aac134aec202da49445919d2f Mon Sep 17 00:00:00 2001 From: Saejin Mahlau-Heinert Date: Wed, 21 Jun 2023 18:45:22 -0500 Subject: [PATCH 12/32] Add completed implementation of concurrent request handling Update configuration file to use `concurrent` instead of max active requests as well, and updates documentation accordingly --- README.md | 29 +++--- example/microwrap.json | 4 +- microwrap/microwrap.py | 213 ++++++++++++++++++----------------------- 3 files changed, 113 insertions(+), 133 deletions(-) diff --git a/README.md b/README.md index 9a226ed..4cfe76c 100644 --- a/README.md +++ b/README.md @@ -6,9 +6,9 @@ MicroWrap is a base container image designed to streamline and simplify the proc # Create an image using microwrap as the base to serve as our runtime image FROM michionlion/microwrap:latest # Configure microwrap -COPY microwrap.json /microwrap.json +COPY example/microwrap.json /microwrap.json # Upload executable to expose as a service -COPY version.sh /version.sh +COPY example/version.sh /version.sh ``` MicroWrap functions as an executable wrapper, abstracting the complexities of network communication and service execution away from the application itself. It consists of an HTTP server that listens for incoming HTTP requests and translates those HTTP requests to command-line invocations of the wrapped executable. The translation process supports parameters -- URL parameters embedded in the request will become `--option value` strings passed to the wrapped executable. The standard output of the wrapped executable will be returned as the body of the response to the triggering request. @@ -33,23 +33,26 @@ To make your application a containerized service, you will need to write a Docke ## Configuration +1. **Host** This is the host name to bind the server to; it defaults to `"0.0.0.0"` and should rarely need to be changed. +1. **Port** This is the port to bind the server to; it defaults to `80` and can be changed if needed. 1. **Executable Path** This is the location of the executable file that will be executed per request. It should be an executable file in your image. -2. **Max Active Requests** This is the number of wrapped-executable invocations to allow at one time; any requests beyond this number will be queued for future invocation. Specify `0` for no limit. -3. **Allowed Parameters** This is a list of URL parameters that will be passed through as command-line options to the wrapped executable. Any other parameters will be ignored. -4. **Default Parameters** This is an object which is mapped to `--attribute value` strings passed to the wrapped executable that can be overridden by URL parameters. Values that are `true` will not map to `"true"`, but instead a value-less `--flag` (for an attribute named `flag`) string; values that are `null`, `false`, or the empty string `""` will cause the parameter to be ignored. +1. **Concurrent** Whether to allow multiple requests to execute invocations concurrently; if `false`, only one invocation will be handled at a time. +1. **Allowed Parameters** This is a list of URL parameters that will be passed through as command-line options to the wrapped executable. Any other parameters will be ignored. +1. **Default Parameters** This is an object which is mapped to `--attribute value` strings passed to the wrapped executable that can be overridden by URL parameters. Values that are `true` will not map to `"true"`, but instead a value-less `--flag` (for an attribute named `flag`) string; values that are `null`, `false`, or the empty string `""` will cause the parameter to be ignored. These configuration parameters should be specified in the `/microwrap.json` configuration file in your image: ```json { - "executablePath": "/root/program.sh", - "maxActiveRequests": 1, - "allowedParameters": ["option1", "flag1"], - "defaultParameters": { - "option1": "defaultValue1", - "flag1": false, - "flag2": true - } + "host": "0.0.0.0", + "port": 8080, + "concurrent": false, + "executablePath": "/root/program.sh", + "allowedParameters": ["option1", "flag1"], + "defaultParameters": { + "option1": "defaultValue1", + "alwaysonflag": true + } } ``` diff --git a/example/microwrap.json b/example/microwrap.json index dee7953..108309a 100644 --- a/example/microwrap.json +++ b/example/microwrap.json @@ -1,6 +1,8 @@ { + "host": "0.0.0.0", + "port": 8080, + "concurrent": false, "executablePath": "/version.sh", - "maxActiveRequests": 0, "allowedParameters": [ "include-build" ], diff --git a/microwrap/microwrap.py b/microwrap/microwrap.py index b2865b4..010990e 100644 --- a/microwrap/microwrap.py +++ b/microwrap/microwrap.py @@ -1,17 +1,14 @@ """MicroWrap.""" +import errno import json import os -import signal import subprocess -import sys import threading import traceback -import wsgiref.simple_server as server -from concurrent.futures import ThreadPoolExecutor -from http.server import BaseHTTPRequestHandler -from queue import Empty, Queue -from typing import Any, Dict, Iterable, List, Union +from socketserver import ThreadingMixIn +from typing import Any, Dict, Iterable, List, Tuple, Union from urllib.parse import parse_qs +from wsgiref.simple_server import WSGIRequestHandler, WSGIServer from wsgiref.types import StartResponse, WSGIEnvironment @@ -26,6 +23,30 @@ def __init__(self, path: str): else: raise ValueError("Invalid configuration file!") + def get_host(self) -> str: + """Return the host to bind to.""" + # TODO: verify valid config + # should be a string + return self.config.get("host", "0.0.0.0") + + def get_port(self) -> int: + """Return the port to bind to.""" + # TODO: verify valid config + # should be an integer and valid port + return self.config.get("port", 80) + + def get_concurrent(self) -> int: + """Return whether to run a multithreaded server.""" + # TODO: verify valid config + # should be a boolean + return self.config.get("concurrent", True) + + def get_executable_path(self) -> str: + """Return the path to the executable.""" + # TODO: verify valid config + # should be an existing path + return self.config.get("executablePath") + def get_allowed_params(self) -> List[str]: """Return the list of allowed query-string parameters.""" # TODO: verify valid config @@ -40,18 +61,6 @@ def get_default_params(self) -> Dict[str, Union[str, bool]]: # if a value is not a str or bool, error should include "should be a string or boolean" return self.config.get("defaultParameters", {}) - def get_executable_path(self) -> str: - """Return the path to the executable.""" - # TODO: verify valid config - # should be an existing path - return self.config.get("executablePath") - - def get_max_active_requests(self) -> int: - """Return the maximum number of active requests.""" - # TODO: verify valid config - # should be a positive integer or 0 (unlimited) - return self.config.get("maxActiveRequests", 1) - def __str__(self): return str(self.config) @@ -64,9 +73,6 @@ def __str__(self): USAGE_HELP = "Usage: microwrap " CONFIG_PATH = "microwrap.json" MAX_THREADS = os.cpu_count() * 4 -RESPONSE_HEADERS = [ - ("Server", f"MicroWrap/1.0.0 {Config(CONFIG_PATH).get_executable_path()}") -] ### @@ -94,6 +100,20 @@ def parse_query_params(config: Config, query_str) -> Dict[str, str]: return params +def get_response_headers(body: str | bytes) -> List[str]: + """Return the response headers derived from the response body.""" + if isinstance(body, str): + body = body.encode() + content = "text/plain" + elif isinstance(body, bytes): + content = "application/octet-stream" + return [ + ("Server", f"MicroWrap/1.0.0 {Config(CONFIG_PATH).get_executable_path()}"), + ("Content-Length", str(len(body))), + ("Content-Type", content), + ] + + ### # Request handler ### @@ -126,86 +146,20 @@ def get_arguments(self): self.arguments.append(value) return self.arguments - def execute(self) -> str: + def execute(self) -> Tuple[str, int]: """Execute the invocation requested.""" - executable = os.path.abspath(self.config.get_executable_path()) - arguments = self.get_arguments() - print(f"{self.get_label()} Executing '{executable}' with args: {arguments}") + cmd = [os.path.abspath(self.config.get_executable_path())] + cmd += self.get_arguments() + print(f"{self.get_label()} Executing '{cmd}'") proc = subprocess.run( - executable=executable, - args=arguments, + cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - check=True, - text=True, + check=False, cwd=".", start_new_session=True, ) - return proc.stdout - - -### -# Threaded WSGI server -### - - -class ThreadedWSGIServer(server.WSGIServer): - """Threaded WSGI Server that uses a ThreadPoolExecutor to handle requests.""" - - def __init__( - self, - host: str, - port: int, - handler: BaseHTTPRequestHandler, - max_workers=None, - ): - super().__init__((host, port), handler, True) - self.requests = Queue() - self.executor = ThreadPoolExecutor(max_workers if max_workers else MAX_THREADS) - self.is_listening = False - - def process_request(self, request, client_address): - """Add requests to the queue instead of handling immediately like super does.""" - self.requests.put((request, client_address)) - print(f"Added request {request} from {client_address} to queue") - print(f"{self.requests.qsize()} requests in queue") - - def serve_forever(self, poll_interval=0.5): - """Continuously handle requests from the queue in separate threads.""" - signal.signal(signal.SIGINT, lambda _s, _f: self.shutdown()) - signal.signal(signal.SIGTERM, lambda _s, _f: self.shutdown()) - self.is_listening = True - print("Starting server...") - while self.is_listening: - try: - print(f"Pulling from queue: {self.requests.queue}") - request, client_address = self.requests.get( - block=True, timeout=poll_interval - ) - print("Retrieved request") - self.executor.submit( - self.process_request_thread, request, client_address - ) - print("Submitted request") - except Empty: - continue - - def process_request_thread(self, request, client_address): - """This function will be executed in a new thread and handle the request.""" - try: - print(f"Processing request {request} from {client_address}") - self.finish_request(request, client_address) - self.shutdown_request(request) - except Exception: - self.handle_error(request, client_address) - self.shutdown_request(request) - - def shutdown(self): - """Stop the server but wait for current requests to complete before exiting.""" - print("Waiting for requests to complete...") - self.is_listening = False - self.executor.shutdown(wait=True, cancel_futures=True) - print("Requests complete. Shutting down...") + return (proc.stdout.decode(), proc.returncode) ### @@ -220,37 +174,58 @@ def microwrap(env: WSGIEnvironment, start_response: StartResponse) -> Iterable[b handler = InvocationRequest(config, env) print(f"{handler.get_label()} Configuration: {config}") print(f"{handler.get_label()} Parameters: {handler.params}") - response_body = handler.execute() - start_response("200 OK", RESPONSE_HEADERS) - return [response_body.encode()] + body, exitcode = handler.execute() + print(f"{handler.get_label()} Finished execution, exit code: {exitcode}") + status = "200 OK" if exitcode == 0 else "500 Internal Server Error" + start_response(status, get_response_headers(body)) + return [body.encode()] except Exception as ex: traceback.print_exception(ex) - start_response("500 Internal Server Error", RESPONSE_HEADERS) - return [str(ex).encode()] + body = f"MicroWrap error: {type(ex)}: {ex}" + start_response("500 Internal Server Error", get_response_headers(body)) + return [body.encode()] + +### +# Infrastructure: WSGI server and main method +### -def run(host="localhost", port=80): + +class ThreadedWSGIServer(ThreadingMixIn, WSGIServer): + """Threaded WSGI Server that uses a thread for each request.""" + + def get_request(self): + while True: + try: + sock, addr = self.socket.accept() + if self.verify_request(sock, addr): + return sock, addr + except OSError as ex: + if ex.errno != errno.EINTR: + raise + + +def run(host="0.0.0.0", port=80, concurrent=True): """Run the WSGI application.""" print(f"Listening on http://{host}:{port}/") - httpd = ThreadedWSGIServer( - host, - port, - server.WSGIRequestHandler, - max_workers=Config(CONFIG_PATH).get_max_active_requests(), - ) + if concurrent: + print("Starting concurrent server...") + httpd = ThreadedWSGIServer((host, port), WSGIRequestHandler) + else: + print("Starting non-concurrent server...") + httpd = WSGIServer((host, port), WSGIRequestHandler) httpd.set_app(microwrap) - httpd.serve_forever(poll_interval=0.5) + try: + httpd.serve_forever(poll_interval=0.5) + except KeyboardInterrupt: + print("\nShutting down...") + httpd.shutdown() if __name__ == "__main__": - if len(sys.argv) == 1: - run() - elif len(sys.argv) == 2: - if sys.argv[1] == "--help" or sys.argv[1] == "-h": - print(USAGE_HELP) - else: - run(sys.argv[1]) - elif len(sys.argv) == 3: - run(sys.argv[1], int(sys.argv[2])) - else: - print(USAGE_HELP) + global_config = Config(CONFIG_PATH) + run( + global_config.get_host(), + global_config.get_port(), + global_config.get_concurrent(), + ) From d4d7e07bc47593b5dcc5bfd065cd07996223ad28 Mon Sep 17 00:00:00 2001 From: Saejin Mahlau-Heinert Date: Thu, 22 Jun 2023 10:24:24 -0500 Subject: [PATCH 13/32] Add Dockerfile update and ignore test files --- .gitignore | 4 ++++ Dockerfile | 9 +++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 65cc7fb..e25dc6c 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,7 @@ build/ *.c *.so + +# Manual test files +/microwrap.json +/test.sh diff --git a/Dockerfile b/Dockerfile index 3e2f283..8b7deaa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,15 +1,16 @@ # Build microwrap -FROM alpine:3.18 as build +FROM alpine:latest as build # Install dependencies +WORKDIR /microwrap RUN apk add --no-cache poetry +COPY . /microwrap RUN poetry install RUN poetry run task compile # Create microwrap image -FROM alpine:3.18 +FROM alpine:latest -COPY --from=build /microwrap /usr/bin/microwrap -COPY --from=build /usr/local/lib/libmicrohttpd.so.12 /usr/local/lib +COPY --from=build /microwrap/build/microwrap /usr/bin/microwrap ENTRYPOINT [ "microwrap" ] From b4276248d9f3d327a007c4b6eb75b0bb520448ef Mon Sep 17 00:00:00 2001 From: Saejin Mahlau-Heinert Date: Thu, 22 Jun 2023 11:02:30 -0500 Subject: [PATCH 14/32] Update pyproject to allow Python 3.11 and up --- poetry.lock | 10 +++++----- pyproject.toml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/poetry.lock b/poetry.lock index a6140be..89bae89 100644 --- a/poetry.lock +++ b/poetry.lock @@ -280,13 +280,13 @@ files = [ [[package]] name = "platformdirs" -version = "3.6.0" +version = "3.7.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." optional = false python-versions = ">=3.7" files = [ - {file = "platformdirs-3.6.0-py3-none-any.whl", hash = "sha256:ffa199e3fbab8365778c4a10e1fbf1b9cd50707de826eb304b50e57ec0cc8d38"}, - {file = "platformdirs-3.6.0.tar.gz", hash = "sha256:57e28820ca8094678b807ff529196506d7a21e17156cb1cddb3e74cebce54640"}, + {file = "platformdirs-3.7.0-py3-none-any.whl", hash = "sha256:cfd065ba43133ff103ab3bd10aecb095c2a0035fcd1f07217c9376900d94ba07"}, + {file = "platformdirs-3.7.0.tar.gz", hash = "sha256:87fbf6473e87c078d536980ba970a472422e94f17b752cfad17024c18876d481"}, ] [package.extras] @@ -468,5 +468,5 @@ files = [ [metadata] lock-version = "2.0" -python-versions = "==3.11.3" -content-hash = "6f52914e88663d00904af8bd59a8246699dfa700d50edff9057b6dc82eabe897" +python-versions = "^3.11" +content-hash = "5dc46b7be75db847dd35ac202cc14f9097d6b2d663ff348470293f9e0df487ae" diff --git a/pyproject.toml b/pyproject.toml index 3c6ea0e..42a7222 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ license = "UNLICENSED" readme = "README.md" [tool.poetry.dependencies] -python = "==3.11.3" +python = "^3.11" bjoern = "^3.2.2" [tool.poetry.group.dev.dependencies] From 92c41bbf6a52565e66c94eb7010edd0aeb9ad601 Mon Sep 17 00:00:00 2001 From: Saejin Mahlau-Heinert Date: Thu, 22 Jun 2023 11:09:13 -0500 Subject: [PATCH 15/32] Add cython dependency installation to Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 8b7deaa..005401d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ FROM alpine:latest as build # Install dependencies WORKDIR /microwrap -RUN apk add --no-cache poetry +RUN apk add --no-cache build-base libev poetry COPY . /microwrap RUN poetry install RUN poetry run task compile From 73b5ace4118c220cf288da8913faf07e975ec3f0 Mon Sep 17 00:00:00 2001 From: Saejin Mahlau-Heinert Date: Thu, 22 Jun 2023 11:35:50 -0500 Subject: [PATCH 16/32] Update example and Dockerfile to use debian instead of alpine --- Dockerfile | 7 ++++--- example/Dockerfile | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 005401d..fe1eda9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,15 +1,16 @@ # Build microwrap -FROM alpine:latest as build +FROM python:3.11-buster as build # Install dependencies WORKDIR /microwrap -RUN apk add --no-cache build-base libev poetry +RUN apt update && apt install -y build-essential libev-dev COPY . /microwrap +RUN pip install poetry RUN poetry install RUN poetry run task compile # Create microwrap image -FROM alpine:latest +FROM python:3.11-slim-buster COPY --from=build /microwrap/build/microwrap /usr/bin/microwrap diff --git a/example/Dockerfile b/example/Dockerfile index f43051c..0ea7c50 100644 --- a/example/Dockerfile +++ b/example/Dockerfile @@ -1,7 +1,7 @@ # Build the service executable in a "development" container -FROM alpine:latest as build +FROM debian:buster as build # Add Java 11 JDK -RUN apk add --no-cache openjdk11 +RUN apt update && apt install -y openjdk-11-jdk # Copy source code to container COPY Version.java /Version.java # Compile source code @@ -10,7 +10,7 @@ RUN javac Version.java # Create an image using microwrap as the base to serve as our runtime image FROM michionlion/microwrap:latest # Install Java 11 JRE -RUN apk add --no-cache openjre11 +RUN apt update && apt install -y openjdk-11-jre && apt clean && rm -rf /var/lib/apt/lists/* # Configure microwrap COPY microwrap.json /microwrap.json # Copy executable (a script to run Version.class) From 8080c5157a80af5b3caefbed98ce976dc6218454 Mon Sep 17 00:00:00 2001 From: Saejin Mahlau-Heinert Date: Thu, 22 Jun 2023 13:13:40 -0400 Subject: [PATCH 17/32] Update example to work --- example/Version.java | 2 -- example/microwrap.json | 2 +- example/version.sh | 10 ++++------ 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/example/Version.java b/example/Version.java index a4cd665..3e93c9f 100644 --- a/example/Version.java +++ b/example/Version.java @@ -1,5 +1,3 @@ -package example; - public class Version { public static void main(String[] args) { var VERSION = "1.0.0"; diff --git a/example/microwrap.json b/example/microwrap.json index 108309a..d91e26f 100644 --- a/example/microwrap.json +++ b/example/microwrap.json @@ -1,6 +1,6 @@ { "host": "0.0.0.0", - "port": 8080, + "port": 80, "concurrent": false, "executablePath": "/version.sh", "allowedParameters": [ diff --git a/example/version.sh b/example/version.sh index e314e36..da58d4c 100755 --- a/example/version.sh +++ b/example/version.sh @@ -2,14 +2,12 @@ USE_JAVA=true -if [ "$USE_JAVA" == "true" && -f Version.class ]; then - java Version.class +if [ "$USE_JAVA" = "true" ] && [ -f Version.class ]; then + java -classpath . Version else - VERSION="1.0.0" - - if [ "$1" == "--include-build" ]; then + VERSION="2.0.0" + if [ "$1" = "--include-build" ]; then VERSION="$VERSION-b4" fi - echo "$VERSION" fi From 4fd7b1362987cb0b1305bbd423c2ff559406eedd Mon Sep 17 00:00:00 2001 From: Saejin Mahlau-Heinert Date: Thu, 22 Jun 2023 12:22:42 -0500 Subject: [PATCH 18/32] Update example to allow variable passing --- example/version.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/version.sh b/example/version.sh index da58d4c..165fab8 100755 --- a/example/version.sh +++ b/example/version.sh @@ -3,7 +3,7 @@ USE_JAVA=true if [ "$USE_JAVA" = "true" ] && [ -f Version.class ]; then - java -classpath . Version + java -classpath . Version "$@" else VERSION="2.0.0" if [ "$1" = "--include-build" ]; then From e92fcc1e93509003b8894386221628abfb2688a9 Mon Sep 17 00:00:00 2001 From: Saejin Mahlau-Heinert Date: Thu, 22 Jun 2023 12:41:10 -0500 Subject: [PATCH 19/32] Update microwrap to log to rotating files --- .gitignore | 2 ++ microwrap/microwrap.py | 47 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index e25dc6c..d2906fa 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,5 @@ build/ # Manual test files /microwrap.json /test.sh +/microwrap.err +/microwrap.log diff --git a/microwrap/microwrap.py b/microwrap/microwrap.py index 010990e..0a54d25 100644 --- a/microwrap/microwrap.py +++ b/microwrap/microwrap.py @@ -1,16 +1,61 @@ """MicroWrap.""" import errno import json +import logging import os import subprocess +import sys import threading import traceback +from logging.handlers import RotatingFileHandler from socketserver import ThreadingMixIn -from typing import Any, Dict, Iterable, List, Tuple, Union +from typing import Any, Dict, Iterable, List, TextIO, Tuple, Union from urllib.parse import parse_qs from wsgiref.simple_server import WSGIRequestHandler, WSGIServer from wsgiref.types import StartResponse, WSGIEnvironment +### +# Logging +### + + +class RotatingLogger: + """A logger that writes to a file and rotates it when it gets too big.""" + + def __init__( + self, + terminal: TextIO, + log_file: str, + max_bytes=20 * 1_000_000, + backup_count=5, + ): + self.terminal = terminal + self.log = RotatingFileHandler( + log_file, maxBytes=max_bytes, backupCount=backup_count + ) + self.log.setFormatter(logging.Formatter("%(message)s")) + + def write(self, message: str): + """Write a message to the log.""" + message = message.strip("\n") + if message: + self.terminal.write(message + "\n") + self.log.emit(logging.makeLogRecord({"msg": message})) + self.log.flush() + + def flush(self): + """Flush the log.""" + self.terminal.flush() + self.log.flush() + + def close(self): + """Close the log.""" + self.log.close() + + +sys.stdout = RotatingLogger(sys.stdout, "microwrap.log") +sys.stderr = RotatingLogger(sys.stderr, "microwrap.err") + class Config: """A MicroWrap configuration file.""" From 30e13253f7de66fb208edab558f596f6d7da2b82 Mon Sep 17 00:00:00 2001 From: Saejin Mahlau-Heinert Date: Thu, 22 Jun 2023 12:46:03 -0500 Subject: [PATCH 20/32] Update pyproject.toml to contain docker-related tasks --- example/Dockerfile | 2 ++ pyproject.toml | 3 +++ 2 files changed, 5 insertions(+) diff --git a/example/Dockerfile b/example/Dockerfile index 0ea7c50..c8bf547 100644 --- a/example/Dockerfile +++ b/example/Dockerfile @@ -17,3 +17,5 @@ COPY microwrap.json /microwrap.json COPY version.sh /version.sh # Copy compiled code from the build image COPY --from=build /Version.class /Version.class + +EXPOSE 80 diff --git a/pyproject.toml b/pyproject.toml index 42a7222..02528cf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,9 @@ get_ldflags = "python -c \"import sysconfig; print(sysconfig.get_config_var('LDF lint = { cmd = "pylint microwrap", help = "check code style with pylint" } pre_compile = { cmd = "mkdir -p build && cython -3 --embed -o build/microwrap.c microwrap/microwrap.py", help = "transpile code to C with cython" } compile = { cmd = "gcc -v -I$({get_includes}) -L$({get_libdir}) -l$({get_lib}) $({get_ldflags}) -o build/microwrap build/microwrap.c", help = "compile code to single executable with cython" } +docker = { cmd = "docker build -t michionlion/microwrap:latest .", help = "build docker image" } +pre_example = { cmd = "docker build -t microwrap-example:latest .", help = "build example docker image", cwd = "example" } +example = { cmd = "docker run --name microwrap-example -p 3000:80 microwrap-example:latest", help = "run example docker image" } [build-system] requires = ["poetry-core"] From 1d87499b39a73d6e997cd9bb48affa2acaead446 Mon Sep 17 00:00:00 2001 From: Saejin Mahlau-Heinert Date: Thu, 22 Jun 2023 13:31:58 -0500 Subject: [PATCH 21/32] Fix issue with using wrong build directory for microwrap-example --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 02528cf..0d3f0bf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,8 +30,8 @@ lint = { cmd = "pylint microwrap", help = "check code style with pylint" } pre_compile = { cmd = "mkdir -p build && cython -3 --embed -o build/microwrap.c microwrap/microwrap.py", help = "transpile code to C with cython" } compile = { cmd = "gcc -v -I$({get_includes}) -L$({get_libdir}) -l$({get_lib}) $({get_ldflags}) -o build/microwrap build/microwrap.c", help = "compile code to single executable with cython" } docker = { cmd = "docker build -t michionlion/microwrap:latest .", help = "build docker image" } -pre_example = { cmd = "docker build -t microwrap-example:latest .", help = "build example docker image", cwd = "example" } -example = { cmd = "docker run --name microwrap-example -p 3000:80 microwrap-example:latest", help = "run example docker image" } +pre_example = { cmd = "docker build -t microwrap-example:latest example", help = "build example docker image" } +example = { cmd = "set -x && docker run --name microwrap-example -p 3000:80 microwrap-example:latest", help = "run example docker image" } [build-system] requires = ["poetry-core"] From e17c5f7d33db16ed1188e5daf5747c9984117301 Mon Sep 17 00:00:00 2001 From: Saejin Mahlau-Heinert Date: Thu, 22 Jun 2023 13:46:40 -0500 Subject: [PATCH 22/32] Ensure we always flush output Sometimes stdout was not flushed until program exit --- microwrap/microwrap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/microwrap/microwrap.py b/microwrap/microwrap.py index 0a54d25..cf2a55e 100644 --- a/microwrap/microwrap.py +++ b/microwrap/microwrap.py @@ -41,7 +41,7 @@ def write(self, message: str): if message: self.terminal.write(message + "\n") self.log.emit(logging.makeLogRecord({"msg": message})) - self.log.flush() + self.flush() def flush(self): """Flush the log.""" From 78243eb8db662a9fe79ef73dd8f9d0af154c6959 Mon Sep 17 00:00:00 2001 From: Saejin Mahlau-Heinert Date: Thu, 22 Jun 2023 15:06:51 -0500 Subject: [PATCH 23/32] Add dockerignore --- .dockerignore | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..fbdb807 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +/build +/.github From 1b276193a4e2013d195251a24d0122b795346bce Mon Sep 17 00:00:00 2001 From: Saejin Mahlau-Heinert Date: Thu, 22 Jun 2023 15:07:08 -0500 Subject: [PATCH 24/32] Add main GitHub Actions workflow --- .github/workflows/main.yml | 56 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 .github/workflows/main.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..a1758b6 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,56 @@ +name: Check Quality + +on: [push, pull_request, workflow_call] + +jobs: + lint: + name: Lint + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - name: Setup Python 3.11 + uses: actions/setup-python@v4 + with: + python-version: "3.11" + - name: Install Poetry + uses: abatilo/actions-poetry@v2 + with: + poetry-version: 1.5.1 + - name: Setup Poetry + run: | + poetry config virtualenvs.create true + poetry config virtualenvs.in-project true + poetry env info + - name: Install dependencies + run: poetry install --no-interaction --no-ansi + - name: Lint code + run: poetry run task lint + - name: Lint writing + uses: actionshub/markdownlint@main + + test: + name: Test + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - name: Setup Python 3.11 + uses: actions/setup-python@v4 + with: + python-version: "3.11" + - name: Install Poetry + uses: abatilo/actions-poetry@v2 + with: + poetry-version: 1.5.1 + - name: Setup Poetry + run: | + poetry config virtualenvs.create true + poetry config virtualenvs.in-project true + poetry env info + - name: Build docker image + run: poetry run task docker + - name: Build example docker image + run: poetry run task pre_example + # - name: Test example docker image + # run: poetry run task test_example From 8c844c8ffa79e81f4dfc04df2ff2736dfec2f05a Mon Sep 17 00:00:00 2001 From: Saejin Mahlau-Heinert Date: Thu, 22 Jun 2023 15:07:38 -0500 Subject: [PATCH 25/32] Add GitHub Actions workflow for publishing to DockerHub --- .github/workflows/publish.yml | 46 +++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 .github/workflows/publish.yml diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..ebdb71e --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,46 @@ +name: Docker + +on: + release: + types: [published] + workflow_dispatch: + +jobs: + test: + uses: ./.github/workflows/main.yml + push: + needs: test + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Log in to Docker Hub + uses: docker/login-action@v2 + with: + username: michionlion + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v4 + with: + images: michionlion/microwrap + tags: | + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + - name: Build and push Docker image + uses: docker/build-push-action@v4 + with: + context: . + file: ./Dockerfile + push: false + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + - name: Debug + run: | + docker ps -a + docker images -a + +# VERSION=$(echo "${{ github.event.release.tag_name }}" | sed -e 's/^v//') +# type=raw,value=latest,enable={{is_default_branch}} From 2e3417b34efecea34c2964854094ef9289b3f9cf Mon Sep 17 00:00:00 2001 From: Saejin Mahlau-Heinert Date: Thu, 22 Jun 2023 15:17:46 -0500 Subject: [PATCH 26/32] Update publish GitHub Actions workflow to use environment --- .github/workflows/publish.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index ebdb71e..1949728 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -11,6 +11,7 @@ jobs: push: needs: test runs-on: ubuntu-latest + environment: dockerhub steps: - name: Checkout uses: actions/checkout@v3 @@ -19,7 +20,7 @@ jobs: - name: Log in to Docker Hub uses: docker/login-action@v2 with: - username: michionlion + username: ${{ vars.DOCKERHUB_USER }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Extract metadata (tags, labels) for Docker id: meta From d20d8d01a9026bf4310139f556bd3720f441d393 Mon Sep 17 00:00:00 2001 From: Saejin Mahlau-Heinert Date: Thu, 22 Jun 2023 15:22:40 -0500 Subject: [PATCH 27/32] Add libev system dependency in GitHub Actions workflow --- .github/workflows/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a1758b6..bc01c89 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -22,6 +22,8 @@ jobs: poetry config virtualenvs.create true poetry config virtualenvs.in-project true poetry env info + - name: Install system dependencies + run: sudo apt install libev-dev - name: Install dependencies run: poetry install --no-interaction --no-ansi - name: Lint code From 20f0484071cf5d2f7851ccc299edcca24e65f875 Mon Sep 17 00:00:00 2001 From: Saejin Mahlau-Heinert Date: Thu, 22 Jun 2023 15:33:52 -0500 Subject: [PATCH 28/32] Add pylint config --- .github/workflows/main.yml | 2 ++ pyproject.toml | 3 +++ 2 files changed, 5 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index bc01c89..9a5f693 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -50,6 +50,8 @@ jobs: poetry config virtualenvs.create true poetry config virtualenvs.in-project true poetry env info + - name: Install dependencies + run: poetry install --no-interaction --no-ansi - name: Build docker image run: poetry run task docker - name: Build example docker image diff --git a/pyproject.toml b/pyproject.toml index 0d3f0bf..0378d75 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,9 @@ pylint = "^2.17.4" taskipy = "^1.11.0" cython = "^0.29.35" +[tool.pylint.'MESSAGES CONTROL'] +disable = ["fixme", "broad-exception-caught"] + [tool.taskipy.settings] use_vars = true From a1702e62292a92418d6fcac2d6e79175955955a2 Mon Sep 17 00:00:00 2001 From: Saejin Mahlau-Heinert Date: Thu, 22 Jun 2023 15:35:39 -0500 Subject: [PATCH 29/32] Add missing system dependency for main GitHub Actions workflow test job --- .github/workflows/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9a5f693..7872c8d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -52,6 +52,8 @@ jobs: poetry env info - name: Install dependencies run: poetry install --no-interaction --no-ansi + - name: Install system dependencies + run: sudo apt install libev-dev - name: Build docker image run: poetry run task docker - name: Build example docker image From a75acab8a580c1543fe890e7baa7fcbe218ae0a8 Mon Sep 17 00:00:00 2001 From: Saejin Mahlau-Heinert Date: Thu, 22 Jun 2023 15:37:29 -0500 Subject: [PATCH 30/32] Remove markdownlint from GitHub Actions workflow --- .github/workflows/main.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7872c8d..f4f073c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -28,8 +28,6 @@ jobs: run: poetry install --no-interaction --no-ansi - name: Lint code run: poetry run task lint - - name: Lint writing - uses: actionshub/markdownlint@main test: name: Test From 9b2318288a5eb6ac0102cdeece40ef29dbe65a8c Mon Sep 17 00:00:00 2001 From: Saejin Mahlau-Heinert Date: Thu, 22 Jun 2023 15:38:40 -0500 Subject: [PATCH 31/32] Fix ordering issue for main.yml GitHub Action --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f4f073c..26f8bc5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -48,10 +48,10 @@ jobs: poetry config virtualenvs.create true poetry config virtualenvs.in-project true poetry env info - - name: Install dependencies - run: poetry install --no-interaction --no-ansi - name: Install system dependencies run: sudo apt install libev-dev + - name: Install dependencies + run: poetry install --no-interaction --no-ansi - name: Build docker image run: poetry run task docker - name: Build example docker image From 9ab39b002ac3dc429b54e3a9780026f4453374d8 Mon Sep 17 00:00:00 2001 From: Saejin Mahlau-Heinert Date: Thu, 22 Jun 2023 15:40:50 -0500 Subject: [PATCH 32/32] Make test job in main GitHub Actions workflow depend on lint --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 26f8bc5..33e4068 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -31,6 +31,7 @@ jobs: test: name: Test + needs: lint runs-on: ubuntu-latest steps: - name: Checkout repository