From ec74d8cea590c534dfa425dd011e2dbf100f322e Mon Sep 17 00:00:00 2001 From: William Chu Date: Wed, 4 Sep 2024 16:06:17 +0800 Subject: [PATCH 1/7] chore: exclude tests from mypy --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 9c5b27a..437f560 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,3 +50,6 @@ style = [ 'py://nitpick/resources/any/commitizen', ] cache = "never" + +[tool.mypy] +exclude = ["tests"] From df9e78039b562b13e9c7f58d0a01287e73cbf1d5 Mon Sep 17 00:00:00 2001 From: William Chu Date: Wed, 4 Sep 2024 16:07:34 +0800 Subject: [PATCH 2/7] chore: bump ruff version in pre-commit --- .pre-commit-config.yaml | 4 ++-- gitops_server/utils/github.py | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3f4a3bc..f25532c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,12 +1,12 @@ repos: - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: v0.2.0 + rev: v0.6.3 hooks: - id: ruff args: - --fix - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: v0.2.0 + rev: v0.6.3 hooks: - id: ruff-format - repo: https://github.com/commitizen-tools/commitizen diff --git a/gitops_server/utils/github.py b/gitops_server/utils/github.py index 813ece0..630fdb4 100644 --- a/gitops_server/utils/github.py +++ b/gitops_server/utils/github.py @@ -8,8 +8,7 @@ GITHUB_OAUTH_TOKEN = os.environ.get("GITHUB_OAUTH_TOKEN") -class IssueNotFound(Exception): - ... +class IssueNotFound(Exception): ... class STATUSES: From 52c2dc5b78fb0be533fd67356e6c7d01fc83ca6a Mon Sep 17 00:00:00 2001 From: William Chu Date: Wed, 4 Sep 2024 16:14:21 +0800 Subject: [PATCH 3/7] chore: add types and remove monkeypatch --- .mise.toml | 2 +- gitops/__init__.py | 1 - gitops/monkeypatches.py | 8 ------ gitops/utils/cli.py | 1 + pyproject.toml | 4 +++ uv.lock | 62 +++++++++++++++++++++++++++++++++++++++++ 6 files changed, 68 insertions(+), 10 deletions(-) delete mode 100644 gitops/monkeypatches.py diff --git a/.mise.toml b/.mise.toml index fca215b..4365931 100644 --- a/.mise.toml +++ b/.mise.toml @@ -29,4 +29,4 @@ run = "helm install --dry-run --debug -f charts/gitops/values.yaml debug charts/ [tasks.release] description = "Bump release versions across all files" -run = "python release.py" \ No newline at end of file +run = "python release.py" diff --git a/gitops/__init__.py b/gitops/__init__.py index 6d291f2..05dc77a 100644 --- a/gitops/__init__.py +++ b/gitops/__init__.py @@ -2,7 +2,6 @@ import sys from pathlib import Path -from . import monkeypatches # NOQA from .utils.cli import success, warning __version__ = "0.11.0" diff --git a/gitops/monkeypatches.py b/gitops/monkeypatches.py deleted file mode 100644 index b891010..0000000 --- a/gitops/monkeypatches.py +++ /dev/null @@ -1,8 +0,0 @@ -""" -REMOVE WHEN: https://github.com/pyinvoke/invoke/commit/8f6c0617c7dc59b105dd1b92fb417e75adc21bea is released. -""" - -import inspect - -if not hasattr(inspect, "getargspec"): - inspect.getargspec = inspect.getfullargspec diff --git a/gitops/utils/cli.py b/gitops/utils/cli.py index 6889d22..2dfd8bc 100644 --- a/gitops/utils/cli.py +++ b/gitops/utils/cli.py @@ -1,4 +1,5 @@ from collections.abc import Callable + from colorama import Fore diff --git a/pyproject.toml b/pyproject.toml index 437f560..c588946 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,10 @@ build-backend = "hatchling.build" dev-dependencies = [ "pytest>=8.3.2", "ruff>=0.6.3", + "types-colorama>=0.4.15.20240311", + "mypy>=1.11.2", + "types-pyyaml>=6.0.12.20240808", + "types-tabulate>=0.9.0.20240106", ] diff --git a/uv.lock b/uv.lock index be35cd8..c38446b 100644 --- a/uv.lock +++ b/uv.lock @@ -231,8 +231,12 @@ server = [ [package.dev-dependencies] dev = [ + { name = "mypy" }, { name = "pytest" }, { name = "ruff" }, + { name = "types-colorama" }, + { name = "types-pyyaml" }, + { name = "types-tabulate" }, ] [package.metadata] @@ -254,8 +258,12 @@ requires-dist = [ [package.metadata.requires-dev] dev = [ + { name = "mypy", specifier = ">=1.11.2" }, { name = "pytest", specifier = ">=8.3.2" }, { name = "ruff", specifier = ">=0.6.3" }, + { name = "types-colorama", specifier = ">=0.4.15.20240311" }, + { name = "types-pyyaml", specifier = ">=6.0.12.20240808" }, + { name = "types-tabulate", specifier = ">=0.9.0.20240106" }, ] [[package]] @@ -383,6 +391,33 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fa/a2/17e1e23c6be0a916219c5292f509360c345b5fa6beeb50d743203c27532c/multidict-6.0.5-py3-none-any.whl", hash = "sha256:0d63c74e3d7ab26de115c49bffc92cc77ed23395303d496eae515d4204a625e7", size = 9729 }, ] +[[package]] +name = "mypy" +version = "1.11.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mypy-extensions" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5c/86/5d7cbc4974fd564550b80fbb8103c05501ea11aa7835edf3351d90095896/mypy-1.11.2.tar.gz", hash = "sha256:7f9993ad3e0ffdc95c2a14b66dee63729f021968bff8ad911867579c65d13a79", size = 3078806 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/35/3a/ed7b12ecc3f6db2f664ccf85cb2e004d3e90bec928e9d7be6aa2f16b7cdf/mypy-1.11.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:e8960dbbbf36906c5c0b7f4fbf2f0c7ffb20f4898e6a879fcf56a41a08b0d318", size = 10990335 }, + { url = "https://files.pythonhosted.org/packages/04/e4/1a9051e2ef10296d206519f1df13d2cc896aea39e8683302f89bf5792a59/mypy-1.11.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:06d26c277962f3fb50e13044674aa10553981ae514288cb7d0a738f495550b36", size = 10007119 }, + { url = "https://files.pythonhosted.org/packages/f3/3c/350a9da895f8a7e87ade0028b962be0252d152e0c2fbaafa6f0658b4d0d4/mypy-1.11.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6e7184632d89d677973a14d00ae4d03214c8bc301ceefcdaf5c474866814c987", size = 12506856 }, + { url = "https://files.pythonhosted.org/packages/b6/49/ee5adf6a49ff13f4202d949544d3d08abb0ea1f3e7f2a6d5b4c10ba0360a/mypy-1.11.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3a66169b92452f72117e2da3a576087025449018afc2d8e9bfe5ffab865709ca", size = 12952066 }, + { url = "https://files.pythonhosted.org/packages/27/c0/b19d709a42b24004d720db37446a42abadf844d5c46a2c442e2a074d70d9/mypy-1.11.2-cp312-cp312-win_amd64.whl", hash = "sha256:969ea3ef09617aff826885a22ece0ddef69d95852cdad2f60c8bb06bf1f71f70", size = 9664000 }, + { url = "https://files.pythonhosted.org/packages/42/3a/bdf730640ac523229dd6578e8a581795720a9321399de494374afc437ec5/mypy-1.11.2-py3-none-any.whl", hash = "sha256:b499bc07dbdcd3de92b0a8b29fdf592c111276f6a12fe29c30f6c417dd546d12", size = 2619625 }, +] + +[[package]] +name = "mypy-extensions" +version = "1.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/98/a4/1ab47638b92648243faf97a5aeb6ea83059cc3624972ab6b8d2316078d3f/mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782", size = 4433 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/e2/5d3f6ada4297caebe1a2add3b126fe800c96f56dbe5d1988a2cbe0b267aa/mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d", size = 4695 }, +] + [[package]] name = "packaging" version = "24.1" @@ -606,6 +641,33 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/40/44/4a5f08c96eb108af5cb50b41f76142f0afa346dfa99d5296fe7202a11854/tabulate-0.9.0-py3-none-any.whl", hash = "sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f", size = 35252 }, ] +[[package]] +name = "types-colorama" +version = "0.4.15.20240311" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/59/73/0fb0b9fe4964b45b2a06ed41b60c352752626db46aa0fb70a49a9e283a75/types-colorama-0.4.15.20240311.tar.gz", hash = "sha256:a28e7f98d17d2b14fb9565d32388e419f4108f557a7d939a66319969b2b99c7a", size = 5608 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/83/6944b4fa01efb2e63ac62b791a8ddf0fee358f93be9f64b8f152648ad9d3/types_colorama-0.4.15.20240311-py3-none-any.whl", hash = "sha256:6391de60ddc0db3f147e31ecb230006a6823e81e380862ffca1e4695c13a0b8e", size = 5840 }, +] + +[[package]] +name = "types-pyyaml" +version = "6.0.12.20240808" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/dd/08/6f5737f645571b7a0b1ebd2fe8b5cf1ee4ec3e707866ca96042a86fc1d10/types-PyYAML-6.0.12.20240808.tar.gz", hash = "sha256:b8f76ddbd7f65440a8bda5526a9607e4c7a322dc2f8e1a8c405644f9a6f4b9af", size = 12359 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f3/ad/ffbad24e2bc8f20bf047ec22af0c0a92f6ce2071eb21c9103df600cda6de/types_PyYAML-6.0.12.20240808-py3-none-any.whl", hash = "sha256:deda34c5c655265fc517b546c902aa6eed2ef8d3e921e4765fe606fe2afe8d35", size = 15298 }, +] + +[[package]] +name = "types-tabulate" +version = "0.9.0.20240106" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cf/9d/65b82ce032fd1cc4df752461175a800c1cfc336461f07ceff10c6a5913eb/types-tabulate-0.9.0.20240106.tar.gz", hash = "sha256:c9b6db10dd7fcf55bd1712dd3537f86ddce72a08fd62bb1af4338c7096ce947e", size = 3442 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f0/17/d53c0bb370100313df6800e9096bdfc27b32b8e4a9390bfb35bc4b17db78/types_tabulate-0.9.0.20240106-py3-none-any.whl", hash = "sha256:0378b7b6fe0ccb4986299496d027a6d4c218298ecad67199bbd0e2d7e9d335a1", size = 3350 }, +] + [[package]] name = "typing-extensions" version = "4.12.2" From 2a7792a6498e5d73cbed7980996d187f37e214c4 Mon Sep 17 00:00:00 2001 From: William Chu Date: Wed, 4 Sep 2024 16:15:59 +0800 Subject: [PATCH 4/7] chore: add type ignores to yaml monkeypatches --- gitops/utils/yaml.py | 8 ++++---- pyproject.toml | 1 + uv.lock | 45 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 4 deletions(-) diff --git a/gitops/utils/yaml.py b/gitops/utils/yaml.py index a044f96..d0f8cc1 100644 --- a/gitops/utils/yaml.py +++ b/gitops/utils/yaml.py @@ -24,12 +24,12 @@ def increase_indent(self, flow=False, indentless=False): if pyyaml.safe_dump is pyyaml.dump: # PyYAML v4.1 SafeDumper = pyyaml.dumper.Dumper - DangerDumper = pyyaml.dumper.DangerDumper + DangerDumper = pyyaml.dumper.DangerDumper # type: ignore else: - SafeDumper = pyyaml.dumper.SafeDumper + SafeDumper = pyyaml.dumper.SafeDumper # type: ignore DangerDumper = pyyaml.dumper.Dumper -SafeDumper.increase_indent = increase_indent +SafeDumper.increase_indent = increase_indent # type: ignore DangerDumper.increase_indent = increase_indent pyyaml.add_representer(dict, map_representer, Dumper=SafeDumper) @@ -43,4 +43,4 @@ def increase_indent(self, flow=False, indentless=False): # Merge PyYAML namespace into ours. # This allows users a drop-in replacement: # import utils.yaml as yaml -from yaml import * # noqa isort:skip +from yaml import * # type: ignore # noqa isort:skip diff --git a/pyproject.toml b/pyproject.toml index c588946..0c37f04 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,7 @@ dev-dependencies = [ "mypy>=1.11.2", "types-pyyaml>=6.0.12.20240808", "types-tabulate>=0.9.0.20240106", + "boto3-stubs>=1.35.11", ] diff --git a/uv.lock b/uv.lock index c38446b..6dac338 100644 --- a/uv.lock +++ b/uv.lock @@ -117,6 +117,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0e/78/d505b8c71139d234e34df1c4a18d0567287494ce63f690337aa2af23219c/boto3-1.34.34-py3-none-any.whl", hash = "sha256:33a8b6d9136fa7427160edb92d2e50f2035f04e9d63a2d1027349053e12626aa", size = 139320 }, ] +[[package]] +name = "boto3-stubs" +version = "1.35.11" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore-stubs" }, + { name = "types-s3transfer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6a/57/adbe2eab12d08f0b259a1c29858c75cc8a9f9486c78a2dcf99f387ce0cb5/boto3_stubs-1.35.11.tar.gz", hash = "sha256:c2d803a9a125648afdda5551e108a59f1ce0d70070b7ef39b27c09699b74735a", size = 90437 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7a/ca/83e8de27ce45bdaf68a773f370f8f7d09d0ae5039f6b85d4a1d6d5ba394f/boto3_stubs-1.35.11-py3-none-any.whl", hash = "sha256:43611ee8fe11402b78241d76a2866086dc836541ef1332bf558f852bf465ac85", size = 61103 }, +] + [[package]] name = "botocore" version = "1.34.162" @@ -131,6 +144,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bc/47/e35f788047c91110f48703a6254e5c84e33111b3291f7b57a653ca00accf/botocore-1.34.162-py3-none-any.whl", hash = "sha256:2d918b02db88d27a75b48275e6fb2506e9adaaddbec1ffa6a8a0898b34e769be", size = 12468049 }, ] +[[package]] +name = "botocore-stubs" +version = "1.35.11" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "types-awscrt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/40/13/59fd2c2625e061cee7baee48eb61d9f9a7c5e847271d0ed0eb3ee250dd40/botocore_stubs-1.35.11.tar.gz", hash = "sha256:af7e1e669c6cf35524a19bb0bb1c18b2bf673cccc9707f129cce02374704047e", size = 40560 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ed/f4/5bcb7b8bbfebaf8d731319b9f64f66163837841c4853e5a31849427cef21/botocore_stubs-1.35.11-py3-none-any.whl", hash = "sha256:e69d1352a230d1c34be91a7ab94679c9fec5148d818c19f31cc4e49da2731296", size = 60133 }, +] + [[package]] name = "certifi" version = "2024.7.4" @@ -231,6 +256,7 @@ server = [ [package.dev-dependencies] dev = [ + { name = "boto3-stubs" }, { name = "mypy" }, { name = "pytest" }, { name = "ruff" }, @@ -258,6 +284,7 @@ requires-dist = [ [package.metadata.requires-dev] dev = [ + { name = "boto3-stubs", specifier = ">=1.35.11" }, { name = "mypy", specifier = ">=1.11.2" }, { name = "pytest", specifier = ">=8.3.2" }, { name = "ruff", specifier = ">=0.6.3" }, @@ -641,6 +668,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/40/44/4a5f08c96eb108af5cb50b41f76142f0afa346dfa99d5296fe7202a11854/tabulate-0.9.0-py3-none-any.whl", hash = "sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f", size = 35252 }, ] +[[package]] +name = "types-awscrt" +version = "0.21.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/61/db/7b9248e4c69f57cfa89336638a3d4b7dc04adfdc9c541b52aef24e4162aa/types_awscrt-0.21.2.tar.gz", hash = "sha256:84a9f4f422ec525c314fdf54c23a1e73edfbcec968560943ca2d41cfae623b38", size = 13225 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6c/71/8c4fcdbc5f41c3567098e05dc28dd0723e945bf441a9d182877f96a94ff9/types_awscrt-0.21.2-py3-none-any.whl", hash = "sha256:0839fe12f0f914d8f7d63ed777c728cb4eccc2d5d79a26e377d12b0604e7bf0e", size = 17550 }, +] + [[package]] name = "types-colorama" version = "0.4.15.20240311" @@ -659,6 +695,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f3/ad/ffbad24e2bc8f20bf047ec22af0c0a92f6ce2071eb21c9103df600cda6de/types_PyYAML-6.0.12.20240808-py3-none-any.whl", hash = "sha256:deda34c5c655265fc517b546c902aa6eed2ef8d3e921e4765fe606fe2afe8d35", size = 15298 }, ] +[[package]] +name = "types-s3transfer" +version = "0.10.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c7/27/27d39076167a007091a75b8596eb2e1c5898986b21c1ec1cdecbda635df0/types_s3transfer-0.10.2.tar.gz", hash = "sha256:60167a3bfb5c536ec6cdb5818f7f9a28edca9dc3e0b5ff85ae374526fc5e576e", size = 13825 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bc/80/24038d0a8850b1a5b104627caaf676e334a01719148e6d6270b9a3c1fd10/types_s3transfer-0.10.2-py3-none-any.whl", hash = "sha256:7a3fec8cd632e2b5efb665a355ef93c2a87fdd5a45b74a949f95a9e628a86356", size = 18739 }, +] + [[package]] name = "types-tabulate" version = "0.9.0.20240106" From e6cc109ef9690957f119781a1695153533305f31 Mon Sep 17 00:00:00 2001 From: William Chu Date: Wed, 4 Sep 2024 16:21:06 +0800 Subject: [PATCH 5/7] chore: remove tasks.py; not used anymore --- README.md | 2 -- tasks.py | 96 ------------------------------------------------------- 2 files changed, 98 deletions(-) delete mode 100644 tasks.py diff --git a/README.md b/README.md index 89fca29..e0ef43d 100644 --- a/README.md +++ b/README.md @@ -40,8 +40,6 @@ This is a git repository that you set up, where you list out all of your applica Secrets should be placed in `secrets.env`. The example file `secrets.example.env` has the environment variables you will need to supply. -Gitops has a helm chart defining its deployment. Invoke scripts are provided to make deployment painless. See `tasks.py`. - Add `export GITOPS_APPS_DIRECTORY=~/` to invoke gitops from any directory. Ensure that gitops has `edit` access to the namespace it is deploying to. An example RoleBinding is: diff --git a/tasks.py b/tasks.py deleted file mode 100644 index c4a3853..0000000 --- a/tasks.py +++ /dev/null @@ -1,96 +0,0 @@ -import json -import os -from base64 import b64encode - -import yaml -from invoke import run, task - - -@task -def test(ctx, pty=True): - run("py.test") - - -@task -def lint(ctx, pty=True): - run("pre-commit run --all-files") - - -@task -def build(ctx): - """Build and push a Docker image to ECR. - - Uses the short hash code for the Git repo to identify this build. This - allows for easier rollback. - """ - local = get_latest_image() - print(f"Building container ({local}) ... ", flush=True) - run(f"docker build -t {local} .") - - -@task -def push(ctx, tag=None): - local = get_latest_image() - remote = get_remote_image(tag) - password = run("aws ecr get-login-password", hide=True, warn=False).stdout.strip() - run(f"docker login -u AWS -p {password} https://{get_repo_uri()}", hide=True) - run(f"docker tag {local} {remote}", hide=False) - print(f"Pushing to ECR ({remote}) ... ", flush=True) - run(f"docker push {remote}", pty=True) - - -@task -def logs(ctx): - name = run( - "kubectl -n default get pods --selector=app=gitops -o jsonpath='{.items[*].metadata.name}'", - hide=True, - ).stdout.strip() - run(f"kubectl -n default logs -f {name}", pty=True) - - -@task -def test_helm(ctx): - """Tests / validates a dry installation of the helm chart""" - run( - "helm install --dry-run --debug --set environment.GIT_CRYPT_KEY_FILE='test' --set" - " environment.CLUSTER_NAME='hi' --set environment.CLUSTER_NAMESPACE='test' debug" - " charts/gitops/" - ) - - -def get_commit_tag(): - return run("git rev-parse --short HEAD", hide=True).stdout.strip() - - -def get_account_id(): - caller_identity = run("aws sts get-caller-identity", hide=True).stdout.strip() - return json.loads(caller_identity)["Account"] - - -def get_repo_uri(): - return f"{get_account_id()}.dkr.ecr.ap-southeast-2.amazonaws.com" - - -def get_latest_image() -> str: - return get_remote_image(tag="latest") - - -def get_remote_image(tag=None) -> str: - tag = tag or get_commit_tag() - return f"{get_repo_uri()}/gitops:{tag}" - - -def get_secret(name): - return b64encode(os.environ[name].encode()).decode() - - -def get_secret_file(name): - data = open(os.environ[name], "rb").read() - return b64encode(data).decode() - - -def get_cluster_name(): - data = run("kubectl config view", hide=True).stdout.strip() - conf = yaml.safe_load(data) - contexts = {c["name"]: c["context"] for c in conf["contexts"]} - return contexts[conf["current-context"]]["cluster"].split(":cluster/")[-1] From 1bd2279365c50f70f8e203daab542e645dc8f5a0 Mon Sep 17 00:00:00 2001 From: William Chu Date: Wed, 4 Sep 2024 16:45:38 +0800 Subject: [PATCH 6/7] chore: all the mypy fixes --- gitops/common/app.py | 2 +- gitops/core.py | 2 +- gitops/db.py | 2 +- gitops/main.py | 2 +- gitops/utils/__init__.py | 8 ++++++-- gitops/utils/apps.py | 4 ++-- gitops/utils/images.py | 2 +- gitops/utils/kube.py | 16 ++++++++-------- gitops_server/logging_config.py | 13 ------------- gitops_server/main.py | 1 - gitops_server/utils/__init__.py | 2 +- gitops_server/utils/slack.py | 9 +++++---- gitops_server/workers/deployer/deploy.py | 11 ++++++----- gitops_server/workers/deployer/hooks.py | 4 ++-- gitops_server/workers/status_updater/worker.py | 6 +++--- 15 files changed, 38 insertions(+), 46 deletions(-) delete mode 100644 gitops_server/logging_config.py diff --git a/gitops/common/app.py b/gitops/common/app.py index 1dee14f..898e00d 100644 --- a/gitops/common/app.py +++ b/gitops/common/app.py @@ -155,7 +155,7 @@ def __init__(self, definition: dict | str): # for backwards compat, any chart definition which is a string, is a git repo self.type = "git" self.git_sha = None - self.git_repo_url = definition + self.git_repo_url = definition or None elif isinstance(definition, dict): self.type = definition["type"] self.git_sha = definition.get("git_sha") diff --git a/gitops/core.py b/gitops/core.py index 07a53e4..aca049c 100644 --- a/gitops/core.py +++ b/gitops/core.py @@ -372,7 +372,7 @@ def git_push(cluster_path: str, retry: int = 3): attempts = 0 while attempts <= retry: result = run(f"cd {cluster_path}; git push", warn=True) - if result.exited == 1 and "remote contains work" in result.stderr: + if result and result.exited == 1 and "remote contains work" in result.stderr: attempts += 1 run(f"cd {cluster_path}; git pull --rebase=true") else: diff --git a/gitops/db.py b/gitops/db.py index abb70c0..9526aa0 100644 --- a/gitops/db.py +++ b/gitops/db.py @@ -6,7 +6,7 @@ import time import boto3 -import dsnparse +import dsnparse # type: ignore[import] from invoke import run, task from .utils import kube diff --git a/gitops/main.py b/gitops/main.py index 989d570..fcf42c7 100644 --- a/gitops/main.py +++ b/gitops/main.py @@ -13,6 +13,6 @@ namespace.add_task(task) # Namespace the rarer ones. -namespace.add_collection(db) +namespace.add_collection(db) # type: ignore program = Program(namespace=namespace, version=version) diff --git a/gitops/utils/__init__.py b/gitops/utils/__init__.py index d68dff0..8b766c8 100644 --- a/gitops/utils/__init__.py +++ b/gitops/utils/__init__.py @@ -13,13 +13,17 @@ def gen_secret(length: int = 64) -> str: def get_account_id() -> str: + # TODO REMOVE THIS FUNCTION and account_id in general + if "ACCOUNT_ID" not in CACHE: # This is not ideal, as it makes an assumption that the account_id of interest is the # one the user is currently sitting in. Ideally, should ask the cluster (though that is # messy in its own right, since we don't necessarily want the cluster that's in the # current context either). - caller_identity = run("aws sts get-caller-identity", hide=True).stdout.strip() - CACHE["ACCOUNT_ID"] = json.loads(caller_identity)["Account"] + result = run("aws sts get-caller-identity", hide=True) + if result: + caller_identity = result.stdout.strip() + CACHE["ACCOUNT_ID"] = json.loads(caller_identity)["Account"] return CACHE["ACCOUNT_ID"] diff --git a/gitops/utils/apps.py b/gitops/utils/apps.py index 168986e..a2b18c5 100644 --- a/gitops/utils/apps.py +++ b/gitops/utils/apps.py @@ -1,5 +1,5 @@ import sys -from pathlib import PosixPath +from pathlib import Path from colorama import Fore from tabulate import tabulate @@ -15,7 +15,7 @@ from .tags import colour_tags, validate_tags -def is_valid_app_directory(directory: PosixPath) -> bool: +def is_valid_app_directory(directory: Path) -> bool: files = ["deployment.yml", "secrets.yml"] file_paths = [(directory / file).is_file() for file in files] return all(file_paths) diff --git a/gitops/utils/images.py b/gitops/utils/images.py index 615cc80..f7d137c 100644 --- a/gitops/utils/images.py +++ b/gitops/utils/images.py @@ -58,7 +58,7 @@ def colour_image(image_tag: str) -> str: return colourise(bits[0], color_hash(bits[0])) -def color_hash(bit: str) -> Fore: # type: ignore[no-untyped-def] +def color_hash(bit: str) -> str: return [ Fore.RED, Fore.GREEN, diff --git a/gitops/utils/kube.py b/gitops/utils/kube.py index 8572d6e..d8e2811 100644 --- a/gitops/utils/kube.py +++ b/gitops/utils/kube.py @@ -20,7 +20,7 @@ import yaml from colorama import Fore from invoke import run -from invoke.exceptions import UnexpectedExit +from invoke.exceptions import Failure, UnexpectedExit from gitops.common.app import App from gitops.settings import get_apps_directory @@ -55,8 +55,8 @@ async def run_job( "image": app.image, "serviceAccountName": app.service_account_name, } - extra_labels = {} - container_resources = {} + extra_labels: dict[str, str] = {} + container_resources: ContainerResources | None = None if cpu and memory: cpu_request = f"{cpu}m" if cpu else "" @@ -275,10 +275,10 @@ async def _find_pod() -> str: _, _, output_log = await async_run(cmd) logs += output_log - except Exception as e: + except Failure as e: if "current phase is Succeeded" not in str(e.result.stdout): raise e - except Exception as e: + except Failure as e: if "current phase is Succeeded" not in str(e.result.stdout): raise e finally: @@ -292,9 +292,9 @@ async def wait_for_pod(context: str, namespace: str, pod: str) -> str | None: while True: cmd = "kubectl get pod" f" --context {context}" f" -n {namespace}" ' -o jsonpath="{.status.phase}"' f" {pod}" stdout, _, _ = await async_run(cmd) - stdout = stdout.decode().lower() - if stdout != "pending": - return stdout + output = stdout.decode().lower() + if output != "pending": + return output def retry(*args: object, max_attempts: int = 3, delay: int = 1): # noqa: C901 diff --git a/gitops_server/logging_config.py b/gitops_server/logging_config.py deleted file mode 100644 index 1af3292..0000000 --- a/gitops_server/logging_config.py +++ /dev/null @@ -1,13 +0,0 @@ -import logging - - -class EndpointFilter(logging.Filter): - def filter(self, record: logging.LogRecord) -> bool: - return record.args[2] != "/" # type: ignore - - -logging_format = "%(asctime)s - %(levelname)s - %(name)s - %(message)s" -logging.basicConfig(format=logging_format, level=logging.INFO) - -# Filter out / from access logs (We don't care about these calls) -logging.getLogger("uvicorn.access").addFilter(EndpointFilter()) diff --git a/gitops_server/main.py b/gitops_server/main.py index 7007aeb..312ac6c 100644 --- a/gitops_server/main.py +++ b/gitops_server/main.py @@ -7,7 +7,6 @@ from gitops_server import settings from gitops_server.app import app -from gitops_server.logging_config import * # noqa from gitops_server.workers import DeploymentStatusWorker, DeployQueueWorker logging.basicConfig(level=logging.INFO) diff --git a/gitops_server/utils/__init__.py b/gitops_server/utils/__init__.py index bfaeaff..7e3fcc9 100644 --- a/gitops_server/utils/__init__.py +++ b/gitops_server/utils/__init__.py @@ -19,7 +19,7 @@ async def run(command, suppress_errors=False) -> RunOutput: ) stdout, stderr = await proc.communicate() - exit_code = proc.returncode + exit_code = proc.returncode or 1 if exit_code == 0: return RunOutput(exit_code=exit_code, output=stdout.decode()) else: diff --git a/gitops_server/utils/slack.py b/gitops_server/utils/slack.py index ffd1bbf..34da80b 100644 --- a/gitops_server/utils/slack.py +++ b/gitops_server/utils/slack.py @@ -37,10 +37,11 @@ async def post(message: str) -> None: logger.info("POSTING TO SLACK") data = {"text": message} async with httpx.AsyncClient() as client: - response = await client.post(SLACK_URL, json=data) - if response.status_code >= 300: - logger.warning("Failed to post a message to slack (see below):") - logger.error(f"{message}", exc_info=True) + if SLACK_URL: + response = await client.post(SLACK_URL, json=data) + if response.status_code >= 300: + logger.warning("Failed to post a message to slack (see below):") + logger.error(f"{message}", exc_info=True) async def find_commiter_slack_user(name: str, email: str) -> SlackUser | None: diff --git a/gitops_server/workers/deployer/deploy.py b/gitops_server/workers/deployer/deploy.py index 066ed28..f4add0d 100644 --- a/gitops_server/workers/deployer/deploy.py +++ b/gitops_server/workers/deployer/deploy.py @@ -33,10 +33,9 @@ async def post_init_summary(source, username, added_apps, updated_apps, removed_ async def post_result(app: App, result: UpdateAppResult, deployer: "Deployer", **kwargs): if result["exit_code"] != 0: - result = await handle_failed_deploy(app, result, deployer) - + deploy_result = await handle_failed_deploy(app, result, deployer) message = ( - result.get("slack_message") + deploy_result["slack_message"] or f"Failed to deploy app `{result['app_name']}` for cluster `{settings.CLUSTER_NAME}`:\n>>>{result['output']}" ) @@ -139,7 +138,8 @@ async def uninstall_app(self, app: App) -> UpdateAppResult: async with self.semaphore: logger.info(f"Uninstalling app {app.name!r}.") result = await run(f"helm uninstall {app.name} -n {app.namespace}", suppress_errors=True) - update_result = UpdateAppResult(app_name=app.name, **result) + if result: + update_result = UpdateAppResult(app_name=app.name, slack_message="", **result) await post_result( app=app, result=update_result, @@ -156,6 +156,7 @@ async def update_app_deployment(self, app: App) -> UpdateAppResult | None: async with self.semaphore: logger.info(f"Deploying app {app.name!r}.") if app.chart.type == "git": + assert app.chart.git_repo_url async with temp_repo(app.chart.git_repo_url, sha=app.chart.git_sha) as chart_folder_path: await run(f"cd {chart_folder_path}; helm dependency build") with tempfile.NamedTemporaryFile(suffix=".yml") as cfg: @@ -195,7 +196,7 @@ async def update_app_deployment(self, app: App) -> UpdateAppResult | None: logger.warning("Local is not implemented yet") return None - update_result = UpdateAppResult(app_name=app.name, **result) + update_result = UpdateAppResult(app_name=app.name, slack_message="", **result) await post_result(app=app, result=update_result, deployer=self) return update_result diff --git a/gitops_server/workers/deployer/hooks.py b/gitops_server/workers/deployer/hooks.py index 9da19d6..6d2c06b 100644 --- a/gitops_server/workers/deployer/hooks.py +++ b/gitops_server/workers/deployer/hooks.py @@ -9,7 +9,7 @@ from gitops_server import settings from gitops_server.types import UpdateAppResult from gitops_server.utils import github -from gitops_server.utils.slack import SlackGroup, find_commiter_slack_user +from gitops_server.utils.slack import SlackGroup, SlackUser, find_commiter_slack_user logger = logging.getLogger(__name__) @@ -74,7 +74,7 @@ async def handle_failed_deploy(app: App, result: UpdateAppResult, deployer) -> U email = deployer.author_email if "devops" in email.lower() or "tickforge" in email.lower(): - slack_user = DEFAULT_USER_GROUP + slack_user: SlackGroup | SlackUser = DEFAULT_USER_GROUP else: slack_user = ( await find_commiter_slack_user(name=deployer.author_name, email=deployer.author_email) or DEFAULT_USER_GROUP diff --git a/gitops_server/workers/status_updater/worker.py b/gitops_server/workers/status_updater/worker.py index 0effda5..4129257 100644 --- a/gitops_server/workers/status_updater/worker.py +++ b/gitops_server/workers/status_updater/worker.py @@ -14,9 +14,9 @@ import asyncio import logging -import kubernetes_asyncio -import kubernetes_asyncio.client -import kubernetes_asyncio.config +import kubernetes_asyncio # type:ignore[import-untyped] +import kubernetes_asyncio.client # type:ignore[import-untyped] +import kubernetes_asyncio.config # type:ignore[import-untyped] from gitops_server.settings import CLUSTER_NAMESPACE from gitops_server.utils import github From 96ee2359efb85e93e46353e64e5f5d4b69cb4ce1 Mon Sep 17 00:00:00 2001 From: William Chu Date: Thu, 5 Sep 2024 07:42:38 +0800 Subject: [PATCH 7/7] chore: remove makefile and add mypy to ci pipeline --- .mise.toml | 2 +- Makefile | 21 --------------------- 2 files changed, 1 insertion(+), 22 deletions(-) delete mode 100644 Makefile diff --git a/.mise.toml b/.mise.toml index 4365931..5f8297c 100644 --- a/.mise.toml +++ b/.mise.toml @@ -22,7 +22,7 @@ run = ["uv run py.test"] run = ["uv run mypy ."] [tasks."ci"] -depends = ["lint", "format", "test"] +depends = ["lint", "format", "test", "mypy"] [tasks."helm:lint"] run = "helm install --dry-run --debug -f charts/gitops/values.yaml debug charts/gitops" diff --git a/Makefile b/Makefile deleted file mode 100644 index 2c744b0..0000000 --- a/Makefile +++ /dev/null @@ -1,21 +0,0 @@ -REPO ?= 305686791668.dkr.ecr.ap-southeast-2.amazonaws.com -ECR_REPO ?= gitops -TAG ?= $(shell git rev-parse --short HEAD) -IMAGE ?= ${REPO}/${ECR_REPO}:${TAG} -LATEST_IMAGE ?= ${REPO}/${ECR_REPO}:latest - -# Print this help message -help: - @echo - @awk '/^#/ {c=substr($$0,3); next} c && /^([a-zA-Z].+):/{ print " \033[32m" $$1 "\033[0m",c }{c=0}' $(MAKEFILE_LIST) |\ - sort |\ - column -s: -t |\ - less -R - -# Test helm chart -helm/lint: - helm install --dry-run --debug -f charts/gitops/values.yaml debug charts/gitops - -# Bump release versions across all files -release: - python release.py \ No newline at end of file