From 547d018a449f1b0c8a819d430c732317837468d2 Mon Sep 17 00:00:00 2001 From: Matthew Pugh Date: Fri, 11 Oct 2024 10:56:00 +0100 Subject: [PATCH] Scaling up and down lambdas (#103) * Scaling up and down lambdas done, but still need to test on infrastructure * Moved lambda to common folder as it will be used across instances. Fixed formatting and a few working issues * Updated to use single lambda and rules to alternate functionality. Added environment parameter for staging to allow force-redeploys ignoring cache * Updated to combine lambdas into one and use cloudwatch events to handle difference. * Updated handler and timeout values * Update infrastructure/environments/cloudformation/full/common/lambda/ecs_scale/README.md Co-authored-by: Christina Moore <30839329+christina-moore@users.noreply.github.com> * Updated testing setup, added actions to auto test lambdas --------- Co-authored-by: Christina Moore <30839329+christina-moore@users.noreply.github.com> --- .github/workflows/build_dagster.yml | 4 +- .github/workflows/build_release.yml | 4 +- .github/workflows/test_lambdas.yml | 47 +++ .../full/common/lambda/__init__.py | 0 .../common/lambda/ecs_scale/.python-version | 1 + .../full/common/lambda/ecs_scale/README.md | 28 ++ .../full/common/lambda/ecs_scale/__init__.py | 0 .../lambda/ecs_scale/ecs_scale/__init__.py | 0 .../lambda/ecs_scale/ecs_scale/handler.py | 49 +++ .../full/common/lambda/ecs_scale/poetry.lock | 278 ++++++++++++++++++ .../common/lambda/ecs_scale/pyproject.toml | 19 ++ .../common/lambda/ecs_scale/tests/__init__.py | 0 .../lambda/ecs_scale/tests/test_handler.py | 66 +++++ .../cloudformation/full/common/scaling.yaml | 113 +++++++ .../cloudformation/full/la/dagster.yaml | 21 ++ .../cloudformation/full/la/s3.yaml | 15 +- 16 files changed, 640 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/test_lambdas.yml create mode 100644 infrastructure/environments/cloudformation/full/common/lambda/__init__.py create mode 100644 infrastructure/environments/cloudformation/full/common/lambda/ecs_scale/.python-version create mode 100644 infrastructure/environments/cloudformation/full/common/lambda/ecs_scale/README.md create mode 100644 infrastructure/environments/cloudformation/full/common/lambda/ecs_scale/__init__.py create mode 100644 infrastructure/environments/cloudformation/full/common/lambda/ecs_scale/ecs_scale/__init__.py create mode 100644 infrastructure/environments/cloudformation/full/common/lambda/ecs_scale/ecs_scale/handler.py create mode 100644 infrastructure/environments/cloudformation/full/common/lambda/ecs_scale/poetry.lock create mode 100644 infrastructure/environments/cloudformation/full/common/lambda/ecs_scale/pyproject.toml create mode 100644 infrastructure/environments/cloudformation/full/common/lambda/ecs_scale/tests/__init__.py create mode 100644 infrastructure/environments/cloudformation/full/common/lambda/ecs_scale/tests/test_handler.py create mode 100644 infrastructure/environments/cloudformation/full/common/scaling.yaml diff --git a/.github/workflows/build_dagster.yml b/.github/workflows/build_dagster.yml index 5a0b9da..2871461 100644 --- a/.github/workflows/build_dagster.yml +++ b/.github/workflows/build_dagster.yml @@ -14,8 +14,8 @@ on: - main jobs: - push_to_registry: - name: Push Docker image to Docker Hub + push_to_staging: + name: Push Latest Docker image to Docker Hub runs-on: ubuntu-latest steps: - name: Check out the repo diff --git a/.github/workflows/build_release.yml b/.github/workflows/build_release.yml index 6ced49a..225f22b 100644 --- a/.github/workflows/build_release.yml +++ b/.github/workflows/build_release.yml @@ -14,8 +14,8 @@ on: - '*' jobs: - push_to_registry: - name: Push Docker image to Docker Hub + push_to_prod: + name: Push Tagged Docker image to Docker Hub runs-on: ubuntu-latest steps: - name: Check out the repo diff --git a/.github/workflows/test_lambdas.yml b/.github/workflows/test_lambdas.yml new file mode 100644 index 0000000..816a22e --- /dev/null +++ b/.github/workflows/test_lambdas.yml @@ -0,0 +1,47 @@ +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +# GitHub recommends pinning actions to a commit SHA. +# To get a newer version, you will need to update the SHA. +# You can also reference a tag or branch, but the action may change without warning. + +name: Publish Docker image +on: + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + test_lambda_functions: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [ "3.11" ] + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install poetry + uses: abatilo/actions-poetry@v2 + - name: Setup a local virtual environment (if no poetry.toml file) + run: | + poetry config virtualenvs.create true --local + poetry config virtualenvs.in-project true --local + - uses: actions/cache@v3 + name: Define a cache for the virtual environment based on the dependencies lock file + with: + path: ./.venv + key: venv-${{ hashFiles('poetry.lock') }} + - name: Install the project dependencies + working-directory: ./infrastructure/environments/cloudformation/full/common/lambda/ecs_scale + run: poetry install + - name: Run the automated tests (for example) + working-directory: ./infrastructure/environments/cloudformation/full/common/lambda/ecs_scale + run: poetry run pytest -v \ No newline at end of file diff --git a/infrastructure/environments/cloudformation/full/common/lambda/__init__.py b/infrastructure/environments/cloudformation/full/common/lambda/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/infrastructure/environments/cloudformation/full/common/lambda/ecs_scale/.python-version b/infrastructure/environments/cloudformation/full/common/lambda/ecs_scale/.python-version new file mode 100644 index 0000000..2419ad5 --- /dev/null +++ b/infrastructure/environments/cloudformation/full/common/lambda/ecs_scale/.python-version @@ -0,0 +1 @@ +3.11.9 diff --git a/infrastructure/environments/cloudformation/full/common/lambda/ecs_scale/README.md b/infrastructure/environments/cloudformation/full/common/lambda/ecs_scale/README.md new file mode 100644 index 0000000..e0f71ed --- /dev/null +++ b/infrastructure/environments/cloudformation/full/common/lambda/ecs_scale/README.md @@ -0,0 +1,28 @@ +# Scaling Lambda + +## Manual Process +To package this, do the following: + +1. If it doesn't exist, make the package directory +```commandline +mkdir package +``` +2. Next, we want to ensure all the libraries are present in the package: +```commandline +poetry run pip install --target ./package boto3 +``` +3. Zip Everything Up +```commandline +cd package +zip -r ../ecs-scaling-lambda.zip +``` +4. Add the lambda function handler to the zip +```commandline +cd ../ecs_scale +zip ../ecs-scaling-lambda.zip ./handler.py +``` +5. The zip should have a flat directory structure, ready to be uploaded to s3 + +## Automated Process +To do. The above process could be done via a github actions and then an upload step could push this to s3, +and redeployed to the lambda. \ No newline at end of file diff --git a/infrastructure/environments/cloudformation/full/common/lambda/ecs_scale/__init__.py b/infrastructure/environments/cloudformation/full/common/lambda/ecs_scale/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/infrastructure/environments/cloudformation/full/common/lambda/ecs_scale/ecs_scale/__init__.py b/infrastructure/environments/cloudformation/full/common/lambda/ecs_scale/ecs_scale/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/infrastructure/environments/cloudformation/full/common/lambda/ecs_scale/ecs_scale/handler.py b/infrastructure/environments/cloudformation/full/common/lambda/ecs_scale/ecs_scale/handler.py new file mode 100644 index 0000000..3dd4609 --- /dev/null +++ b/infrastructure/environments/cloudformation/full/common/lambda/ecs_scale/ecs_scale/handler.py @@ -0,0 +1,49 @@ +import boto3 +import os +import logging + + +def lambda_handler(event, context): + # Will be fed by the event rule if the pipeline is to be turned on or off + if event["pipelineStatus"] == "true": + pipeline_count = 1 + elif event["pipelineStatus"] == "false": + pipeline_count = 0 + + # On staging systems, we want to make sure to reload the pipeline every time. + # On production systems, we should use cached version unless there's an infrastructure update + force_new_deployment = False if os.environ.get("ENVIRONMENT") == "prod" else True + + if logging.getLogger().hasHandlers(): + # The Lambda environment pre-configures a handler logging to stderr. If a handler is already configured, + # `.basicConfig` does not execute. Thus we set the level directly. + logging.getLogger().setLevel(logging.INFO) + else: + logging.basicConfig(level=logging.INFO) + + logger = logging.getLogger("ecs_scaler") + + ecs_client = boto3.client("ecs") + cluster = os.environ.get("ECS_CLUSTER_NAME") + dagit_service = os.environ.get("ECS_DAGIT_SERVICE_NAMES") + daemon_service = os.environ.get("ECS_DAEMON_SERVICE_NAMES") + code_service = os.environ.get("ECS_CODE_SERVER_SERVICE_NAMES") + + for service in (dagit_service, daemon_service, code_service): + try: + response = ecs_client.update_service( + cluster=cluster, + service=service, + desiredCount=pipeline_count, + forceNewDeployment=force_new_deployment, + ) + logger.info( + f"Successfully scaled Service {service} to {pipeline_count}: {response}" + ) + except Exception as e: + logger.error(f"Could not Scale Service {service} to {pipeline_count}: {e}") + continue + + +if __name__ == "__main__": + print(lambda_handler(None)) diff --git a/infrastructure/environments/cloudformation/full/common/lambda/ecs_scale/poetry.lock b/infrastructure/environments/cloudformation/full/common/lambda/ecs_scale/poetry.lock new file mode 100644 index 0000000..1cbe302 --- /dev/null +++ b/infrastructure/environments/cloudformation/full/common/lambda/ecs_scale/poetry.lock @@ -0,0 +1,278 @@ +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. + +[[package]] +name = "black" +version = "24.8.0" +description = "The uncompromising code formatter." +optional = false +python-versions = ">=3.8" +files = [ + {file = "black-24.8.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:09cdeb74d494ec023ded657f7092ba518e8cf78fa8386155e4a03fdcc44679e6"}, + {file = "black-24.8.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:81c6742da39f33b08e791da38410f32e27d632260e599df7245cccee2064afeb"}, + {file = "black-24.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:707a1ca89221bc8a1a64fb5e15ef39cd755633daa672a9db7498d1c19de66a42"}, + {file = "black-24.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:d6417535d99c37cee4091a2f24eb2b6d5ec42b144d50f1f2e436d9fe1916fe1a"}, + {file = "black-24.8.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:fb6e2c0b86bbd43dee042e48059c9ad7830abd5c94b0bc518c0eeec57c3eddc1"}, + {file = "black-24.8.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:837fd281f1908d0076844bc2b801ad2d369c78c45cf800cad7b61686051041af"}, + {file = "black-24.8.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:62e8730977f0b77998029da7971fa896ceefa2c4c4933fcd593fa599ecbf97a4"}, + {file = "black-24.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:72901b4913cbac8972ad911dc4098d5753704d1f3c56e44ae8dce99eecb0e3af"}, + {file = "black-24.8.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:7c046c1d1eeb7aea9335da62472481d3bbf3fd986e093cffd35f4385c94ae368"}, + {file = "black-24.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:649f6d84ccbae73ab767e206772cc2d7a393a001070a4c814a546afd0d423aed"}, + {file = "black-24.8.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2b59b250fdba5f9a9cd9d0ece6e6d993d91ce877d121d161e4698af3eb9c1018"}, + {file = "black-24.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:6e55d30d44bed36593c3163b9bc63bf58b3b30e4611e4d88a0c3c239930ed5b2"}, + {file = "black-24.8.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:505289f17ceda596658ae81b61ebbe2d9b25aa78067035184ed0a9d855d18afd"}, + {file = "black-24.8.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b19c9ad992c7883ad84c9b22aaa73562a16b819c1d8db7a1a1a49fb7ec13c7d2"}, + {file = "black-24.8.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f13f7f386f86f8121d76599114bb8c17b69d962137fc70efe56137727c7047e"}, + {file = "black-24.8.0-cp38-cp38-win_amd64.whl", hash = "sha256:f490dbd59680d809ca31efdae20e634f3fae27fba3ce0ba3208333b713bc3920"}, + {file = "black-24.8.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:eab4dd44ce80dea27dc69db40dab62d4ca96112f87996bca68cd75639aeb2e4c"}, + {file = "black-24.8.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3c4285573d4897a7610054af5a890bde7c65cb466040c5f0c8b732812d7f0e5e"}, + {file = "black-24.8.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9e84e33b37be070ba135176c123ae52a51f82306def9f7d063ee302ecab2cf47"}, + {file = "black-24.8.0-cp39-cp39-win_amd64.whl", hash = "sha256:73bbf84ed136e45d451a260c6b73ed674652f90a2b3211d6a35e78054563a9bb"}, + {file = "black-24.8.0-py3-none-any.whl", hash = "sha256:972085c618ee94f402da1af548a4f218c754ea7e5dc70acb168bfaca4c2542ed"}, + {file = "black-24.8.0.tar.gz", hash = "sha256:2500945420b6784c38b9ee885af039f5e7471ef284ab03fa35ecdde4688cd83f"}, +] + +[package.dependencies] +click = ">=8.0.0" +mypy-extensions = ">=0.4.3" +packaging = ">=22.0" +pathspec = ">=0.9.0" +platformdirs = ">=2" + +[package.extras] +colorama = ["colorama (>=0.4.3)"] +d = ["aiohttp (>=3.7.4)", "aiohttp (>=3.7.4,!=3.9.0)"] +jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] +uvloop = ["uvloop (>=0.15.2)"] + +[[package]] +name = "boto3" +version = "1.35.1" +description = "The AWS SDK for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "boto3-1.35.1-py3-none-any.whl", hash = "sha256:9fe1a97de136a89d2957e41e3591a7648a6e08d63165255b4cc69c5ba1e2a177"}, + {file = "boto3-1.35.1.tar.gz", hash = "sha256:9028dd814c7f06bc92822d1fabdd8b68fe39b5993da5ca7acd7f7c6f7c7c332f"}, +] + +[package.dependencies] +botocore = ">=1.35.1,<1.36.0" +jmespath = ">=0.7.1,<2.0.0" +s3transfer = ">=0.10.0,<0.11.0" + +[package.extras] +crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] + +[[package]] +name = "botocore" +version = "1.35.1" +description = "Low-level, data-driven core of boto 3." +optional = false +python-versions = ">=3.8" +files = [ + {file = "botocore-1.35.1-py3-none-any.whl", hash = "sha256:bce42967d0f03b79cf25b2b6a36221fb2fb15f98e6fa4155b66b672ab192013b"}, + {file = "botocore-1.35.1.tar.gz", hash = "sha256:e599cef6305e950a212f85adb86c1f4713d4b2678b3c2ec54df6b8e2dbe9ef2f"}, +] + +[package.dependencies] +jmespath = ">=0.7.1,<2.0.0" +python-dateutil = ">=2.1,<3.0.0" +urllib3 = {version = ">=1.25.4,<2.2.0 || >2.2.0,<3", markers = "python_version >= \"3.10\""} + +[package.extras] +crt = ["awscrt (==0.21.2)"] + +[[package]] +name = "click" +version = "8.1.7" +description = "Composable command line interface toolkit" +optional = false +python-versions = ">=3.7" +files = [ + {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, + {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "jmespath" +version = "1.0.1" +description = "JSON Matching Expressions" +optional = false +python-versions = ">=3.7" +files = [ + {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, + {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, +] + +[[package]] +name = "mypy-extensions" +version = "1.0.0" +description = "Type system extensions for programs checked with the mypy type checker." +optional = false +python-versions = ">=3.5" +files = [ + {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, + {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, +] + +[[package]] +name = "packaging" +version = "24.1" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "packaging-24.1-py3-none-any.whl", hash = "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124"}, + {file = "packaging-24.1.tar.gz", hash = "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002"}, +] + +[[package]] +name = "pathspec" +version = "0.12.1" +description = "Utility library for gitignore style pattern matching of file paths." +optional = false +python-versions = ">=3.8" +files = [ + {file = "pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08"}, + {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"}, +] + +[[package]] +name = "platformdirs" +version = "4.2.2" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.2-py3-none-any.whl", hash = "sha256:2d7a1657e36a80ea911db832a8a6ece5ee53d8de21edd5cc5879af6530b1bfee"}, + {file = "platformdirs-4.2.2.tar.gz", hash = "sha256:38b7b51f512eed9e84a22788b4bce1de17c0adb134d6becb09836e37d8654cd3"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] +type = ["mypy (>=1.8)"] + +[[package]] +name = "pluggy" +version = "1.5.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, + {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "pytest" +version = "8.3.3" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-8.3.3-py3-none-any.whl", hash = "sha256:a6853c7375b2663155079443d2e45de913a911a11d669df02a50814944db57b2"}, + {file = "pytest-8.3.3.tar.gz", hash = "sha256:70b98107bd648308a7952b06e6ca9a50bc660be218d53c257cc1fc94fda10181"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=1.5,<2" + +[package.extras] +dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, + {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "s3transfer" +version = "0.10.2" +description = "An Amazon S3 Transfer Manager" +optional = false +python-versions = ">=3.8" +files = [ + {file = "s3transfer-0.10.2-py3-none-any.whl", hash = "sha256:eca1c20de70a39daee580aef4986996620f365c4e0fda6a86100231d62f1bf69"}, + {file = "s3transfer-0.10.2.tar.gz", hash = "sha256:0711534e9356d3cc692fdde846b4a1e4b0cb6519971860796e6bc4c7aea00ef6"}, +] + +[package.dependencies] +botocore = ">=1.33.2,<2.0a.0" + +[package.extras] +crt = ["botocore[crt] (>=1.33.2,<2.0a.0)"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "urllib3" +version = "2.2.2" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.2-py3-none-any.whl", hash = "sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472"}, + {file = "urllib3-2.2.2.tar.gz", hash = "sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[metadata] +lock-version = "2.0" +python-versions = "^3.11" +content-hash = "dda179e12cca71e7048261d71811a6e837ec85f26a8bf9f91d39eb9b66f5c48f" diff --git a/infrastructure/environments/cloudformation/full/common/lambda/ecs_scale/pyproject.toml b/infrastructure/environments/cloudformation/full/common/lambda/ecs_scale/pyproject.toml new file mode 100644 index 0000000..d5c9bf9 --- /dev/null +++ b/infrastructure/environments/cloudformation/full/common/lambda/ecs_scale/pyproject.toml @@ -0,0 +1,19 @@ +[tool.poetry] +name = "ecs_scale" +version = "0.1.0" +description = "A lambda to scale down ECS services for the data platform" +authors = ["Matthew Pugh "] +license = "MIT" + +[tool.poetry.dependencies] +python = "^3.11" +boto3 = "^1.35.0" + + +[tool.poetry.group.dev.dependencies] +black = "^24.8.0" +pytest = "^8.3.3" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/infrastructure/environments/cloudformation/full/common/lambda/ecs_scale/tests/__init__.py b/infrastructure/environments/cloudformation/full/common/lambda/ecs_scale/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/infrastructure/environments/cloudformation/full/common/lambda/ecs_scale/tests/test_handler.py b/infrastructure/environments/cloudformation/full/common/lambda/ecs_scale/tests/test_handler.py new file mode 100644 index 0000000..80988ad --- /dev/null +++ b/infrastructure/environments/cloudformation/full/common/lambda/ecs_scale/tests/test_handler.py @@ -0,0 +1,66 @@ +import os +import unittest +from unittest.mock import patch, Mock, call, MagicMock +from ecs_scale.handler import lambda_handler +import logging + + +class TestLambdaHandler(unittest.TestCase): + @patch("os.environ.get") + @patch("boto3.client") + def test_lambda_handler(self, mock_boto_client, mock_get_env): + # Mock environment variables + mock_get_env.side_effect = ( + "prod", # Return 'stag' for 'ENVIRONMENT' + "my-cluster", # Return 'my-cluster' for 'ECS_CLUSTER_NAME' + "dagit-service", # Return 'dagit-service' for 'ECS_DAGIT_SERVICE_NAMES' + "daemon-service", # Return 'daemon-service' for 'ECS_DAEMON_SERVICE_NAMES' + "code-server-service", # Return 'code-server-service' for 'ECS_CODE_SERVER_SERVICE_NAMES' + ) + + mock_ecs_client = Mock() + mock_boto_client.return_value = mock_ecs_client + + # Mock ecs_client.update_service with a Mock object + mock_update_service = Mock(return_value={"some": "response"}) + + # Invoke the Lambda function with a sample event (optional) + event = { + "pipelineStatus": "false" + } # You can provide a sample event here if needed + + lambda_handler(event, None) # Pass context=None for mocking + + # Assertions to verify function behavior + mock_get_env.assert_has_calls( + [ + call("ENVIRONMENT"), + call("ECS_CLUSTER_NAME"), + call("ECS_DAGIT_SERVICE_NAMES"), + call("ECS_DAEMON_SERVICE_NAMES"), + call("ECS_CODE_SERVER_SERVICE_NAMES"), + ] + ) + + mock_ecs_client.update_service.assert_has_calls( + [ + call( + cluster="my-cluster", + service="dagit-service", + desiredCount=0, + forceNewDeployment=False, + ), + call( + cluster="my-cluster", + service="daemon-service", + desiredCount=0, + forceNewDeployment=False, + ), + call( + cluster="my-cluster", + service="code-server-service", + desiredCount=0, + forceNewDeployment=False, + ), + ] + ) diff --git a/infrastructure/environments/cloudformation/full/common/scaling.yaml b/infrastructure/environments/cloudformation/full/common/scaling.yaml new file mode 100644 index 0000000..ff436c1 --- /dev/null +++ b/infrastructure/environments/cloudformation/full/common/scaling.yaml @@ -0,0 +1,113 @@ +AWSTemplateFormatVersion: '2010-09-09' +Description: ECS Service with Auto-Scaling + + +Parameters: + DaemonServiceName: + Type: String + Description: Name for the Dagster Daemon Service + DaemonServiceARN: + Type: String + Description: ARN for the Dagster Daemon Service + DagitServiceName: + Type: String + Description: Name for the Dagit Service + DagitServiceARN: + Type: String + Description: ARN for the Dagit Service + CodeServerServiceName: + Type: String + Description: Name for the Dagit Service + CodeServerServiceARN: + Type: String + Description: ARN for the Dagit Service + ECSClusterName: + Type: String + Description: Name for the ECS Dagster Service + TimeOn: + Type: String + Description: Time to turn the pipeline on (e.g., 23 for 11pm) + Default: 23 + TimeOff: + Type: String + Description: Time to turn the pipeline off (e.g., 23 for 11pm, 5 for 5am) + Default: 5 + Environment: + Type: String + Description: | + Determines the type of environment. "stag" and "prod" are the two valid strings. Stag will auto-deploy + new versions, while prod will only deploy the cached versions and updates will need to be applied through + infrastructure updates. + Default: prod + OrganisationName: + Type: String + Description: Name of the organisation running this service + ProjectName: + Type: String + Description: Name of the application + Default: fons + LambdaCodeBucket: + Type: String + Description: Name of the bucket that stores the lambda code + +Resources: + CloudWatchEventsTurnOnRule: + Type: AWS::Events::Rule + Properties: + Name: DagsterTurnOnRule + ScheduleExpression: !Sub "cron(0 ${TimeOn} ? * MON-FRI *)" + Targets: + - Id: 'ScalingLambda' + Arn: !GetAtt FonsScalingLambda.Arn + Input: '{ "pipelineStatus": "true" }' + + CloudWatchEventsTurnOffRule: + Type: AWS::Events::Rule + Properties: + Name: DagsterTurnOffRule + ScheduleExpression: !Sub "cron(0 ${TimeOff} ? * MON-FRI *)" + Targets: + - Id: 'ScalingLambda' + Arn: !GetAtt FonsScalingLambda.Arn + Input: '{ "pipelineStatus": "false" }' + + FonsDagsterScalingLambdaRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: lambda.amazonaws.com + Action: 'sts:AssumeRole' + Policies: + - PolicyName: ECSScalingPolicy + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - ecs:UpdateService + Resource: + - !Ref DaemonServiceARN + - !Ref DagitServiceARN + - !Ref CodeServerServiceARN + + FonsScalingLambda: + Type: AWS::Lambda::Function + Properties: + Code: + S3Bucket: !Ref LambdaCodeBucket + S3Key: ecs-scaling-lambda.zip + Role: !GetAtt FonsDagsterScalingLambdaRole.Arn + Handler: handler.lambda_handler + Runtime: python3.11 + Timeout: 60 + Environment: + Variables: + ECS_CLUSTER_NAME: !Ref ECSClusterName + ECS_DAGIT_SERVICE_NAMES: !Ref DagitServiceName + ECS_DAEMON_SERVICE_NAMES: !Ref DaemonServiceName + ECS_CODE_SERVER_SERVICE_NAMES: !Ref CodeServerServiceName + ENVIRONMENT: !Ref Environment \ No newline at end of file diff --git a/infrastructure/environments/cloudformation/full/la/dagster.yaml b/infrastructure/environments/cloudformation/full/la/dagster.yaml index 403b48a..fa76e6b 100644 --- a/infrastructure/environments/cloudformation/full/la/dagster.yaml +++ b/infrastructure/environments/cloudformation/full/la/dagster.yaml @@ -773,3 +773,24 @@ Outputs: DatabaseEndpointAddress: Description: Endpoint Address for the Database Value: !GetAtt DagsterDatabaseCluster.Endpoint.Address + ECSClusterName: + Description: ARN for the ECS Cluster running dagster + Value: !Ref DagsterCluster + DaemonServiceName: + Description: Name for the Dagster Daemon Service + Value: !GetAtt DagsterDaemonService.Name + DaemonServiceARN: + Description: ARN for the Dagster Daemon Service + Value: !Ref DagsterDaemonService + DagitServiceName: + Description: Name for the Dagit Service + Value: !GetAtt DagitService.Name + DagitServiceARN: + Description: ARN for the Dagit Service + Value: !Ref DagitService + CodeServerServiceName: + Description: Name for the Dagit Service + Value: !GetAtt CodeServerService.Name + CodeServerServiceARN: + Description: ARN for the Dagit Service + Value: !Ref CodeServerService diff --git a/infrastructure/environments/cloudformation/full/la/s3.yaml b/infrastructure/environments/cloudformation/full/la/s3.yaml index fc72c29..37ad178 100644 --- a/infrastructure/environments/cloudformation/full/la/s3.yaml +++ b/infrastructure/environments/cloudformation/full/la/s3.yaml @@ -401,6 +401,16 @@ Resources: - - !GetAtt SharedBucket.Arn - "/*" + LambdaCodeBucket: + Type: 'AWS::S3::Bucket' + Properties: + BucketName: !Sub "${AppName}-lambda-code-${OrganisationName}-${Environment}" + AccessControl: Private + BucketEncryption: + ServerSideEncryptionConfiguration: + - ServerSideEncryptionByDefault: + SSEAlgorithm: AES256 + Outputs: IngressBucketName: Value: !Ref DataStoreBucket @@ -428,4 +438,7 @@ Outputs: Description: Arn of the shared Amazon S3 bucket with a lifecycle configuration. SharedRoleARN: Value: !GetAtt SharedStorageRole.Arn - Description: ARN of the role used to access the data store bucket \ No newline at end of file + Description: ARN of the role used to access the data store bucket + LambdaCodeBucket: + Value: !Ref LambdaCodeBucket + Description: Name of the Code Bucket to store lambda functions \ No newline at end of file