From fbc0c42245c76658aae1beb71fc033903408560c Mon Sep 17 00:00:00 2001 From: srinivasreddych Date: Mon, 19 Aug 2024 17:34:48 -0500 Subject: [PATCH] refactored docker images replication module --- .../versions/default.yaml | 1 + .../CODE_OF_CONDUCT.md | 4 - .../dockerimage-replication/CONTRIBUTING.md | 59 ------------- .../dockerimage-replication/LICENSE | 14 --- .../dockerimage-replication/delete-repos.py | 28 +----- .../dockerimage-replication/deployspec.yaml | 2 +- .../dockerimage-replication/modulestack.yaml | 2 +- .../dockerimage-replication/pyproject.toml | 12 +-- .../dockerimage-replication/replication.sh | 88 +++++++++++++------ 9 files changed, 76 insertions(+), 134 deletions(-) delete mode 100755 modules/replication/dockerimage-replication/CODE_OF_CONDUCT.md delete mode 100755 modules/replication/dockerimage-replication/CONTRIBUTING.md delete mode 100755 modules/replication/dockerimage-replication/LICENSE diff --git a/data/eks_dockerimage-replication/versions/default.yaml b/data/eks_dockerimage-replication/versions/default.yaml index 68ae32db..f843c786 100644 --- a/data/eks_dockerimage-replication/versions/default.yaml +++ b/data/eks_dockerimage-replication/versions/default.yaml @@ -11,6 +11,7 @@ charts: tag: location: chart path: appVersion + prefix: v alb_controller: name: aws-load-balancer-controller repository: "https://aws.github.io/eks-charts" diff --git a/modules/replication/dockerimage-replication/CODE_OF_CONDUCT.md b/modules/replication/dockerimage-replication/CODE_OF_CONDUCT.md deleted file mode 100755 index 5b627cfa..00000000 --- a/modules/replication/dockerimage-replication/CODE_OF_CONDUCT.md +++ /dev/null @@ -1,4 +0,0 @@ -## Code of Conduct -This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). -For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact -opensource-codeofconduct@amazon.com with any additional questions or comments. diff --git a/modules/replication/dockerimage-replication/CONTRIBUTING.md b/modules/replication/dockerimage-replication/CONTRIBUTING.md deleted file mode 100755 index c4b6a1c5..00000000 --- a/modules/replication/dockerimage-replication/CONTRIBUTING.md +++ /dev/null @@ -1,59 +0,0 @@ -# Contributing Guidelines - -Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional -documentation, we greatly value feedback and contributions from our community. - -Please read through this document before submitting any issues or pull requests to ensure we have all the necessary -information to effectively respond to your bug report or contribution. - - -## Reporting Bugs/Feature Requests - -We welcome you to use the GitHub issue tracker to report bugs or suggest features. - -When filing an issue, please check existing open, or recently closed, issues to make sure somebody else hasn't already -reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: - -* A reproducible test case or series of steps -* The version of our code being used -* Any modifications you've made relevant to the bug -* Anything unusual about your environment or deployment - - -## Contributing via Pull Requests -Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: - -1. You are working against the latest source on the *main* branch. -2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. -3. You open an issue to discuss any significant work - we would hate for your time to be wasted. - -To send us a pull request, please: - -1. Fork the repository. -2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. -3. Ensure local tests pass. -4. Commit to your fork using clear commit messages. -5. Send us a pull request, answering any default questions in the pull request interface. -6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. - -GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and -[creating a pull request](https://help.github.com/articles/creating-a-pull-request/). - - -## Finding contributions to work on -Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any 'help wanted' issues is a great place to start. - - -## Code of Conduct -This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). -For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact -opensource-codeofconduct@amazon.com with any additional questions or comments. - - -## Security issue notifications -If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. - - -## Licensing - -See the [LICENSE](LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. diff --git a/modules/replication/dockerimage-replication/LICENSE b/modules/replication/dockerimage-replication/LICENSE deleted file mode 100755 index 6aa0c458..00000000 --- a/modules/replication/dockerimage-replication/LICENSE +++ /dev/null @@ -1,14 +0,0 @@ -Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software is furnished to do so. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/modules/replication/dockerimage-replication/delete-repos.py b/modules/replication/dockerimage-replication/delete-repos.py index 1505a376..17974ee1 100644 --- a/modules/replication/dockerimage-replication/delete-repos.py +++ b/modules/replication/dockerimage-replication/delete-repos.py @@ -2,6 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 #!/usr/bin/env python +import sys + import boto3 from botocore.exceptions import ClientError @@ -25,27 +27,5 @@ def cleanup_ecr_repos(prefix: str) -> None: raise ex -def cleanup_ssm_params(path: str) -> None: - paginator = SSM_CLIENT.get_paginator("get_parameters_by_path") - - response_iterator = paginator.paginate(Path=path) - - for page in response_iterator: - for entry in page["Parameters"]: - print("Deleting the SSM PARAMETER: {}".format(entry["Name"])) - try: - SSM_CLIENT.delete_parameter(Name=entry["Name"]) - except ClientError as ex: - if ex.response["Error"]["Code"] == "ParameterNotFound": - print("SSM PARAMETER: {} is not found, no action needed".format(entry["Name"])) - continue - else: - raise ex - - -# Cleanups ECR Repositories based on a prefix -cleanup_ecr_repos(prefix="idf") - -# Cleanups SSM Parameters based on a specific path pattern -# cleanup_ssm_params(path="/idf/eks/chart") -# cleanup_ssm_params(path="/idf/eks/ami") +# Cleanups ECR Repositories based on a prefix - ProjectName +cleanup_ecr_repos(prefix=sys.argv[1]) diff --git a/modules/replication/dockerimage-replication/deployspec.yaml b/modules/replication/dockerimage-replication/deployspec.yaml index 2b9c595e..a0e551fe 100644 --- a/modules/replication/dockerimage-replication/deployspec.yaml +++ b/modules/replication/dockerimage-replication/deployspec.yaml @@ -19,7 +19,7 @@ deploy: fi sleep 15 fi - - python3 get-list-of-eks-images.py --eks-version ${SEEDFARMER_PARAMETER_EKS_VERSION} --versions-directory data/eks_dockerimage-replication/versions --update-helm-repos --registry-prefix "${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/${AWS_CODESEEDER_NAME}-" + - python3 get-list-of-eks-images.py --eks-version ${SEEDFARMER_PARAMETER_EKS_VERSION} --versions-directory data/eks_dockerimage-replication/versions --update-helm-repos --registry-prefix "${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/${SEEDFARMER_PROJECT_NAME}-" - chmod +x replication.sh - bash replication.sh create - aws s3 cp replication-result.json s3://${S3_BUCKET_NAME}/${S3_OBJECT_NAME} diff --git a/modules/replication/dockerimage-replication/modulestack.yaml b/modules/replication/dockerimage-replication/modulestack.yaml index 976e9eaf..ef155702 100644 --- a/modules/replication/dockerimage-replication/modulestack.yaml +++ b/modules/replication/dockerimage-replication/modulestack.yaml @@ -55,5 +55,5 @@ Resources: Resource: - !Sub "arn:${AWS::Partition}:s3:::*-d*-rep*-*/*" Version: 2012-10-17 - PolicyName: "idf-modulespecific-policy" + PolicyName: "modulespecific-policy" Roles: [!Ref RoleName] \ No newline at end of file diff --git a/modules/replication/dockerimage-replication/pyproject.toml b/modules/replication/dockerimage-replication/pyproject.toml index f8c094ef..8c920ebe 100644 --- a/modules/replication/dockerimage-replication/pyproject.toml +++ b/modules/replication/dockerimage-replication/pyproject.toml @@ -1,32 +1,32 @@ [tool.ruff] +line-length = 120 +target-version = "py38" exclude = [ ".eggs", ".git", ".hg", ".mypy_cache", - ".ruff_cache", ".tox", ".venv", + ".env", "_build", "buck-out", "build", "dist", "codeseeder", ] -line-length = 120 -target-version = "py37" [tool.ruff.lint] select = ["F", "I", "E", "W"] fixable = ["ALL"] + [tool.mypy] -python_version = "3.7" +python_version = 3.8 strict = true ignore_missing_imports = true -disallow_untyped_decorators = false +allow_untyped_decorators = true exclude = "codeseeder.out/|example/|tests/" -warn_unused_ignores = false [tool.pytest.ini_options] addopts = "-v --cov=. --cov-report term" diff --git a/modules/replication/dockerimage-replication/replication.sh b/modules/replication/dockerimage-replication/replication.sh index af704d9e..77901212 100755 --- a/modules/replication/dockerimage-replication/replication.sh +++ b/modules/replication/dockerimage-replication/replication.sh @@ -5,47 +5,85 @@ set -euo pipefail set +x +# Check AWS Domain +if [[ $AWS_PARTITION == "aws" ]]; then + export AWS_DOMAIN="amazonaws.com" +else + export AWS_DOMAIN="amazonaws.com.cn" +fi + +# Login to ECR +login_to_ecr() { + local account_id="$1" + aws ecr get-login-password --region "$AWS_DEFAULT_REGION" | docker login --username AWS --password-stdin "$account_id.dkr.ecr.$AWS_DEFAULT_REGION.$AWS_DOMAIN" +} + +# check if the repository exists +repository_exists() { + local repo_name="$1" + aws ecr describe-repositories --repository-names "$repo_name" > /dev/null 2>&1 +} + +# create an AWS ECR +create_repository() { + local repo_name="$1" + aws ecr create-repository --repository-name "$repo_name" --image-scanning-configuration scanOnPush=true + sleep 10 +} + +# pull and push images +pull_and_push_image() { + local image="$1" + local target_ecr_tag="$2" + docker pull "$image" + docker tag "$image" "$target_ecr_tag" + docker push "$target_ecr_tag" + docker rmi "$image" +} + +# check if the container image exists +image_exists() { + local repo_name="$1" + local image_tag="$2" + aws ecr batch-get-image --repository-name "$repo_name" --image-ids "imageTag=$image_tag" --query 'images[].imageId.imageTag' --output text | grep -q "$image_tag" +} + create() { + # Iterate through the list of images while IFS="" read -r image || [ -n "$image" ] do image_name=$(echo $image | awk -F ':' '{ print $1 }') image_tag=$(echo $image | awk -F ':' '{ print $2 }') if ( [[ ${image_name} =~ ^[0-9] ]] ); then IMAGE_ACCOUNT_ID=$(echo $image_name | awk -F '/' '{print $1}' | awk -F '.' '{print $1}') - aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $IMAGE_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com + login_to_ecr $IMAGE_ACCOUNT_ID fi - TARGET_REPOSITORY_NAME=${AWS_CODESEEDER_NAME}-${image_name} - TARGET_ECR_TAG=$AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$TARGET_REPOSITORY_NAME:${image_tag} - IMAGE_META="$( aws ecr batch-get-image --repository-name=$TARGET_REPOSITORY_NAME --image-ids=imageTag=$image_tag --query 'images[].imageId.imageTag' --output text )" || true - if [[ $IMAGE_META == $image_tag ]]; then - echo "$IMAGE_META found in $TARGET_ECR_TAG skipping replication" + TARGET_REPOSITORY_NAME=${SEEDFARMER_PROJECT_NAME}-${image_name} + TARGET_ECR_TAG=$AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.${AWS_DOMAIN}/$TARGET_REPOSITORY_NAME:${image_tag} + + if ! repository_exists "$TARGET_REPOSITORY_NAME"; then + echo "$TARGET_REPOSITORY_NAME not found in ECR. Creating..." + create_repository "$TARGET_REPOSITORY_NAME" else + echo "$TARGET_REPOSITORY_NAME found in ECR" + fi + + login_to_ecr $AWS_ACCOUNT_ID + + if ! image_exists "$TARGET_REPOSITORY_NAME" "$image_tag"; then echo "$TARGET_REPOSITORY_NAME:$image_tag not found, fetching" - echo Pulling $image - docker pull $image - # Setting connection with AWS ECR - DESCRIBE_REPO=$(aws ecr describe-repositories --repository-names $TARGET_REPOSITORY_NAME || echo "REPOSITORY_MISSING" ) - if [[ $DESCRIBE_REPO == "REPOSITORY_MISSING" ]]; then - echo "$TARGET_REPOSITORY_NAME not found in ECR. Creating..." - aws ecr create-repository --repository-name $TARGET_REPOSITORY_NAME --image-scanning-configuration scanOnPush=true - sleep 10 - else - echo "$TARGET_REPOSITORY_NAME found in ECR" - fi - aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com - # Tagging and pushing Docker images according to https://docs.aws.amazon.com/AmazonECR/latest/userguide/docker-pull-ecr-image.html - docker tag $image $TARGET_ECR_TAG - docker push $TARGET_ECR_TAG - # Deleting so it wouldn't cause issues with codebuild storage space for huge images - docker rmi $image + pull_and_push_image "$image" "$TARGET_ECR_TAG" + else + echo "$image_tag found in $TARGET_ECR_TAG skipping replication" fi + done < images.txt } destroy() { echo "WARNING: The destroy workflow removes the ECR repositories which we were created during replication" - python delete-repos.py + python delete-repos.py ${SEEDFARMER_PROJECT_NAME} } -$1 +"$@"