chore: refactored docker images replication module

CHANGELOG.md

@@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - replace deprecated `auto_delete_images` with `empty_on_delete` &  bump CDK version in `ecr` module
 - fixed the failing regex error in the eks module when env var SEEDFARMER_PARAMETER_REPLICATED_ECR_IMAGES_METADATA_S3_PATH is not set
+- refactored docker images replication module
 
 ### **Removed**
 

data/eks_dockerimage-replication/versions/default.yaml

@@ -11,6 +11,7 @@ charts:
         tag:
           location: chart
           path: appVersion
+          prefix: v
   alb_controller:
     name: aws-load-balancer-controller
     repository: "https://aws.github.io/eks-charts"

modules/replication/dockerimage-replication/delete-repos.py

@@ -2,6 +2,8 @@
 # SPDX-License-Identifier: Apache-2.0
 
 #!/usr/bin/env python
+import sys
+
 import boto3
 from botocore.exceptions import ClientError
 
@@ -25,27 +27,5 @@ def cleanup_ecr_repos(prefix: str) -> None:
                         raise ex
 
 
-def cleanup_ssm_params(path: str) -> None:
-    paginator = SSM_CLIENT.get_paginator("get_parameters_by_path")
-
-    response_iterator = paginator.paginate(Path=path)
-
-    for page in response_iterator:
-        for entry in page["Parameters"]:
-            print("Deleting the SSM PARAMETER: {}".format(entry["Name"]))
-            try:
-                SSM_CLIENT.delete_parameter(Name=entry["Name"])
-            except ClientError as ex:
-                if ex.response["Error"]["Code"] == "ParameterNotFound":
-                    print("SSM PARAMETER: {} is not found, no action needed".format(entry["Name"]))
-                    continue
-                else:
-                    raise ex
-
-
-# Cleanups ECR Repositories based on a prefix
-cleanup_ecr_repos(prefix="idf")
-
-# Cleanups SSM Parameters based on a specific path pattern
-# cleanup_ssm_params(path="/idf/eks/chart")
-# cleanup_ssm_params(path="/idf/eks/ami")
+# Cleanups ECR Repositories based on a prefix - ProjectName
+cleanup_ecr_repos(prefix=sys.argv[1])

modules/replication/dockerimage-replication/deployspec.yaml

@@ -19,7 +19,7 @@ deploy:
             fi
             sleep 15
           fi
-        - python3 get-list-of-eks-images.py --eks-version ${SEEDFARMER_PARAMETER_EKS_VERSION} --versions-directory data/eks_dockerimage-replication/versions --update-helm-repos --registry-prefix "${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/${AWS_CODESEEDER_NAME}-"
+        - python3 get-list-of-eks-images.py --eks-version ${SEEDFARMER_PARAMETER_EKS_VERSION} --versions-directory data/eks_dockerimage-replication/versions --update-helm-repos --registry-prefix "${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/${SEEDFARMER_PROJECT_NAME}-"        
         - chmod +x replication.sh
         - bash replication.sh create
         - aws s3 cp replication-result.json s3://${S3_BUCKET_NAME}/${S3_OBJECT_NAME}

modules/replication/dockerimage-replication/get-list-of-eks-images.py

@@ -97,7 +97,7 @@ def main() -> None:
 
                         if registry:
                             custom_chart_values[workload]["values"] = parser.add_branch_to_dict(
-                                custom_chart_values[workload]["values"],  # type: ignore
+                                custom_chart_values[workload]["values"],
                                 v,
                                 f"{args.registry_prefix}{registry}",
                             )
@@ -118,7 +118,7 @@ def main() -> None:
                             repository_in_chart_values = f"{args.registry_prefix}{repository}"
 
                         custom_chart_values[workload]["values"] = parser.add_branch_to_dict(
-                            custom_chart_values[workload]["values"],  # type: ignore
+                            custom_chart_values[workload]["values"],
                             v,
                             repository_in_chart_values,
                         )
@@ -129,7 +129,7 @@ def main() -> None:
                         tag = parser.parse_value(parsed_charts[workload], values, image_name, v, k)
 
                         custom_chart_values[workload]["values"] = parser.add_branch_to_dict(
-                            custom_chart_values[workload]["values"],  # type: ignore[arg-type]
+                            custom_chart_values[workload]["values"],
                             v,
                             tag,
                         )
@@ -141,7 +141,7 @@ def main() -> None:
                     # set value to empty, e.g. digest as it will be different after push to ECR
                     if k == "remove":
                         custom_chart_values[workload]["values"] = parser.add_branch_to_dict(
-                            custom_chart_values[workload]["values"],  # type: ignore[arg-type]
+                            custom_chart_values[workload]["values"],
                             v,
                             "",
                         )

modules/replication/dockerimage-replication/modulestack.yaml

@@ -55,5 +55,5 @@ Resources:
             Resource:
               - !Sub "arn:${AWS::Partition}:s3:::*-d*-rep*-*/*"
         Version: 2012-10-17
-      PolicyName: "idf-modulespecific-policy"
+      PolicyName: "modulespecific-policy"
       Roles: [!Ref RoleName]

modules/replication/dockerimage-replication/pyproject.toml

@@ -1,32 +1,32 @@
 [tool.ruff]
+line-length = 120
+target-version = "py38"
 exclude = [
     ".eggs",
     ".git",
     ".hg",
     ".mypy_cache",
-    ".ruff_cache",
     ".tox",
     ".venv",
+    ".env",
     "_build",
     "buck-out",
     "build",
     "dist",
     "codeseeder",
 ]
-line-length = 120
-target-version = "py37"
 
 [tool.ruff.lint]
 select = ["F", "I", "E", "W"]
 fixable = ["ALL"]
 
+
 [tool.mypy]
-python_version = "3.7"
+python_version = 3.8
 strict = true
 ignore_missing_imports = true
-disallow_untyped_decorators = false
+allow_untyped_decorators = true
 exclude = "codeseeder.out/|example/|tests/"
-warn_unused_ignores = false
 
 [tool.pytest.ini_options]
 addopts = "-v --cov=. --cov-report term"

modules/replication/dockerimage-replication/replication.sh

@@ -5,47 +5,85 @@
 set -euo pipefail
 set +x
 
+# Check AWS Domain
+if [[ $AWS_PARTITION == "aws" ]]; then
+    export AWS_DOMAIN="amazonaws.com"
+else
+    export AWS_DOMAIN="amazonaws.com.cn"
+fi
+
+# Login to ECR
+login_to_ecr() {
+    local account_id="$1"
+    aws ecr get-login-password --region "$AWS_DEFAULT_REGION" | docker login --username AWS --password-stdin "$account_id.dkr.ecr.$AWS_DEFAULT_REGION.$AWS_DOMAIN"
+}
+
+# check if the repository exists
+repository_exists() {
+    local repo_name="$1"
+    aws ecr describe-repositories --repository-names "$repo_name" > /dev/null 2>&1
+}
+
+# create an AWS ECR
+create_repository() {
+    local repo_name="$1"
+    aws ecr create-repository --repository-name "$repo_name" --image-scanning-configuration scanOnPush=true
+    sleep 10
+}
+
+# pull and push images
+pull_and_push_image() {
+    local image="$1"
+    local target_ecr_tag="$2"
+    docker pull "$image"
+    docker tag "$image" "$target_ecr_tag"
+    docker push "$target_ecr_tag"
+    docker rmi "$image"
+}
+
+# check if the container image exists
+image_exists() {
+    local repo_name="$1"
+    local image_tag="$2"
+    aws ecr batch-get-image --repository-name "$repo_name" --image-ids "imageTag=$image_tag" --query 'images[].imageId.imageTag' --output text | grep -q "$image_tag"
+}
+
 create() {
+    # Iterate through the list of images
     while IFS="" read -r image || [ -n "$image" ]
     do
     image_name=$(echo $image | awk -F ':' '{ print $1 }')
     image_tag=$(echo $image | awk -F ':' '{ print $2 }')
     if ( [[ ${image_name} =~ ^[0-9] ]] ); then
         IMAGE_ACCOUNT_ID=$(echo $image_name | awk -F '/' '{print $1}' | awk -F '.' '{print $1}')
-        aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $IMAGE_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com
+        login_to_ecr $IMAGE_ACCOUNT_ID
     fi
 
-    TARGET_REPOSITORY_NAME=${AWS_CODESEEDER_NAME}-${image_name}
-    TARGET_ECR_TAG=$AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$TARGET_REPOSITORY_NAME:${image_tag}
-    IMAGE_META="$( aws ecr batch-get-image --repository-name=$TARGET_REPOSITORY_NAME --image-ids=imageTag=$image_tag --query 'images[].imageId.imageTag' --output text )" || true
-    if [[ $IMAGE_META == $image_tag ]]; then
-        echo "$IMAGE_META found in $TARGET_ECR_TAG skipping replication"
+    TARGET_REPOSITORY_NAME=${SEEDFARMER_PROJECT_NAME}-${image_name}
+    TARGET_ECR_TAG=$AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.${AWS_DOMAIN}/$TARGET_REPOSITORY_NAME:${image_tag}
+
+    if ! repository_exists "$TARGET_REPOSITORY_NAME"; then
+        echo "$TARGET_REPOSITORY_NAME not found in ECR. Creating..."
+        create_repository "$TARGET_REPOSITORY_NAME"
     else
+        echo "$TARGET_REPOSITORY_NAME found in ECR"
+    fi
+
+    login_to_ecr $AWS_ACCOUNT_ID
+
+    if ! image_exists "$TARGET_REPOSITORY_NAME" "$image_tag"; then
         echo "$TARGET_REPOSITORY_NAME:$image_tag not found, fetching"
-        echo Pulling $image
-        docker pull $image
-        # Setting connection with AWS ECR
-        DESCRIBE_REPO=$(aws ecr describe-repositories --repository-names $TARGET_REPOSITORY_NAME || echo "REPOSITORY_MISSING" )
-        if [[ $DESCRIBE_REPO == "REPOSITORY_MISSING" ]]; then
-            echo "$TARGET_REPOSITORY_NAME not found in ECR. Creating..."
-            aws ecr create-repository --repository-name $TARGET_REPOSITORY_NAME --image-scanning-configuration scanOnPush=true
-            sleep 10
-        else
-            echo "$TARGET_REPOSITORY_NAME found in ECR"
-        fi
-        aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com
-        # Tagging and pushing Docker images according to https://docs.aws.amazon.com/AmazonECR/latest/userguide/docker-pull-ecr-image.html
-        docker tag $image $TARGET_ECR_TAG
-        docker push $TARGET_ECR_TAG
-        # Deleting so it wouldn't cause issues with codebuild storage space for huge images
-        docker rmi $image
+        pull_and_push_image "$image" "$TARGET_ECR_TAG"
+    else
+        echo "$image_tag found in $TARGET_ECR_TAG skipping replication"
     fi
+
     done < images.txt
 }
 
 destroy() {
     echo "WARNING: The destroy workflow removes the ECR repositories which we were created during replication"
-    python delete-repos.py
+    python delete-repos.py ${SEEDFARMER_PROJECT_NAME}
 }
 
-$1
+"$@"