From 3e9e55a848b8c1884ee0add31f236685e237e12b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Pierre?= Date: Thu, 5 Sep 2024 13:59:24 +1200 Subject: [PATCH] [Update] build: better checks --- Makefile | 101 ++++++++++++++++++++++++++++++---- src/py/extra/features/cors.py | 5 +- src/py/extra/http/model.py | 6 +- src/py/extra/server.py | 2 +- src/py/extra/utils/htmpl.py | 4 +- src/py/extra/utils/json.py | 6 +- 6 files changed, 104 insertions(+), 20 deletions(-) diff --git a/Makefile b/Makefile index aa20177..583946a 100644 --- a/Makefile +++ b/Makefile @@ -13,13 +13,40 @@ MODULES_PY:=$(filter-out %/__main__,$(filter-out %/__init__,$(SOURCES_PY:$(PATH_ PATH_LOCAL_PY=$(firstword $(shell python -c "import sys,pathlib;sys.stdout.write(' '.join([_ for _ in sys.path if _.startswith(str(pathlib.Path.home()))] ))")) PATH_LOCAL_BIN=$(HOME)/.local/bin +REQUIRE_PY=flake8 bandit mypy +PREP_ALL=$(REQUIRE_PY:%=build/require-py-%.task) +# Commands +BANDIT=python -m bandit +FLAKE8=python -m flake8 +MYPY=python -m mypy +MYPYC=mypyc + +cmd-check=if ! $$(which $1 &> /dev/null ); then echo "ERR Could not find command $1"; exit 1; fi; $1 + +.PHONY: prep +prep: $(PREP_ALL) + @ + +.PHONY: run +run: + @ + +.PHONY: ci +ci: check test + @ + + -audit: require-py-bandit - bandit -r $(PATH_SOURCES_PY) + + +.PHONY: audit +audit: check-bandit + @echo "=== $@" # NOTE: The compilation seems to create many small modules instead of a big single one +.PHONY: compile compile: - @ + @echo "=== $@" echo "Compiling $(MODULES_PY): $(SOURCES_PY)" # NOTE: Output is going to be like 'extra/__init__.cpython-310-x86_64-linux-gnu.so' @@ -27,20 +54,68 @@ compile: $(foreach M,$(MODULES_PY),mkdir -p build/$M;) env -C build MYPYPATH=$(realpath .)/src/py mypyc -p extra -check: lint - @ +.PHONY: check +check: check-bandit check-flakes check-strict + echo "=== $@" +.PHONY: check-compiled check-compiled: - @COMPILED=$$(PYTHONPATH=build python -c "import extra;print(extra)") + @ + echo "=== $@" + COMPILED=$$(PYTHONPATH=build python -c "import extra;print(extra)") echo "Extra compiled at: $$COMPILED" +.PHONY: check-bandit +check-bandit: $(PREP_ALL) + @echo "=== $@" + $(BANDIT) -r -s B101 src/py/coda + +.PHONY: check-flakes +check-flakes: $(PREP_ALL) + @echo "=== $@" + $(FLAKE8) --ignore=E1,E203,E302,E401,E501,E704,E741,E266,F821,W $(SOURCES_PY) -lint: - @flake8 --ignore=E1,E202,E203,E231,E227,E252,E302,E401,E501,E704,E741,F821,W $(SOURCES_PY) +.PHONY: check-mypyc +check-mypyc: $(PREP_ALL) + @$(call cmd-check,mypyc) $(SOURCES_PY) +.PHONY: check-strict +check-strict: $(PREP_ALL) + @ + count_ok=0 + count_err=0 + files_err= + for item in $(SOURCES_PY); do + if $(MYPY) --strict $$item; then + count_ok=$$(($$count_ok+1)) + else + count_err=$$(($$count_err+1)) + files_err+=" $$item" + fi + done + summary="OK $$count_ok ERR $$count_err TOTAL $$(($$count_err + $$count_ok))" + if [ "$$count_err" != "0" ]; then + for item in $$files_err; do + echo "ERR $$item" + done + echo "EOS FAIL $$summary" + exit 1 + else + echo "EOS OK $$summary" + fi + + + + +.PHONY: lint +lint: check-flakes + @ + +.PHONY: format format: @black $(SOURCES_PY) +.PHONY: install install: @for file in $(SOURCES_BIN); do echo "Installing $(PATH_LOCAL_BIN)/$$(basename $$file)" @@ -60,6 +135,7 @@ install: fi +.PHONY: try-install try-uninstall: @for file in $(SOURCES_BIN); do unlink $(PATH_LOCAL_BIN)/$$(basename $$file) @@ -70,8 +146,12 @@ try-uninstall: done fi -require-py-%: - @if [ -z "$$(which '$*' 2> /dev/null)" ]; then $(PYTHON) -mpip install --user --upgrade '$*'; fi +build/require-py-%.task: + @ + if $(PYTHON) -mpip install --user --upgrade '$*'; then + mkdir -p "$(dir $@)" + touch "$@" + fi data/csic_2010-normalTrafficTraining.txt: curl -o "$@" 'https://gitlab.fing.edu.uy/gsi/web-application-attacks-datasets/-/raw/master/csic_2010/normalTrafficTest.txt?inline=false' @@ -82,7 +162,6 @@ data/csic_2010-anomalousTrafficTraining.txt: print-%: $(info $*=$($*)) -.PHONY: audit .ONESHELL: # EOF # diff --git a/src/py/extra/features/cors.py b/src/py/extra/features/cors.py index e91408f..bc560f5 100644 --- a/src/py/extra/features/cors.py +++ b/src/py/extra/features/cors.py @@ -15,7 +15,10 @@ def cors( def setCORSHeaders( - request: HTTPRequest | HTTPResponse, *, origin=None, allowAll=True + request: HTTPRequest | HTTPResponse, + *, + origin: str | None = None, + allowAll: bool = True ) -> HTTPResponse: """Takes the given request or response, and return (a response) with the CORS headers set properly. diff --git a/src/py/extra/http/model.py b/src/py/extra/http/model.py index 2ffdea1..4e0a700 100644 --- a/src/py/extra/http/model.py +++ b/src/py/extra/http/model.py @@ -233,7 +233,7 @@ def cookies(self) -> Iterable[str]: for _ in self._cookies.keys(): yield _ - def cookie(self, name: str) -> Morsel | None: + def cookie(self, name: str) -> Morsel[str] | None: return self._cookies.get(name) # FIXME: Should be header @@ -291,7 +291,7 @@ def respond( # API # ========================================================================= - def __str__(self): + def __str__(self) -> str: return f"Request({self.method} {self.path}{f'?{self.query}' if self.query else ''} {self.headers})" @@ -432,7 +432,7 @@ def head(self) -> bytes: # TODO: UTF8 maybe? Why ASCII? return "\r\n".join(lines).encode("ascii") - def __str__(self): + def __str__(self) -> str: return f"Response({self.protocol} {self.status} {self.message} {self.headers} {self.body})" diff --git a/src/py/extra/server.py b/src/py/extra/server.py index f1f139d..687ac42 100644 --- a/src/py/extra/server.py +++ b/src/py/extra/server.py @@ -19,7 +19,7 @@ class ServerOptions(NamedTuple): - host: str = "0.0.0.0" + host: str = "0.0.0.0" # nosec: B104 port: int = 8000 backlog: int = 10_000 timeout: float = 10.0 diff --git a/src/py/extra/utils/htmpl.py b/src/py/extra/utils/htmpl.py index 2a38e54..f69ff9c 100644 --- a/src/py/extra/utils/htmpl.py +++ b/src/py/extra/utils/htmpl.py @@ -169,7 +169,9 @@ def f(*children: TNodeContent, **attributes: TAttributeContent): ( v if isinstance(v, list) - else list(v) if isinstance(v, tuple) else [v] + else ( + [_ for _ in cast(tuple, v)] if isinstance(v, tuple) else [v] + ) ), ) elif k == "_": diff --git a/src/py/extra/utils/json.py b/src/py/extra/utils/json.py index e294e1f..6d32b7f 100644 --- a/src/py/extra/utils/json.py +++ b/src/py/extra/utils/json.py @@ -1,11 +1,11 @@ -from typing import Any +from typing import Any, TypeAlias, cast import json as basejson from .primitives import asPrimitive # TODO: We do want to use ORJSON when available: https://github.com/tktech/json_benchmark -TJSON = None | int | float | bool | list | dict +TJSON: TypeAlias = None | int | float | bool | list[Any] | dict[str, Any] def json(value: Any) -> bytes: @@ -15,7 +15,7 @@ def json(value: Any) -> bytes: def unjson(value: bytes | str) -> TJSON: """Converts JSON-encoded to a string.""" - return basejson.loads(value) + return cast(TJSON, basejson.loads(value)) # EOF