Merge pull request #5 from abk7777/refactor/local-makefiles

Add Lambda trigger, refactor by service using Makefiles, add SSM and Secrets

Author: chrisammon3000

.gitignore

@@ -94,7 +94,6 @@ _M*
 _*.yaml
 _*.json
 logs.txt
-config/
 logs/
 0.1-gfe-sandbox.ipynb 
 0.1-dev-build-graph.ipynb
@@ -106,4 +105,7 @@ build/Makefile
 load/Makefile
 .aws-sam
 packaged.yaml
-samconfig.toml
+samconfig.toml
+scripts/
+cfn/
+_pr.md

Makefile

@@ -1,69 +1,105 @@
 ##########################
 # Bootstrapping variables
 ##########################
-
-# TODO: move to config file and parse with jq, or use SSM parameter store
 export STAGE ?= dev
 export APP_NAME ?= gfe-db
-export NEO4J_USERNAME ?= neo4j
-export NEO4J_PASSWORD ?= gfedb
+export AWS_ACCOUNT ?= $(shell aws sts get-caller-identity --query Account --output text)
 export REGION ?= us-east-1
 
+export DATA_BUCKET_NAME ?= ${STAGE}-${APP_NAME}-${AWS_ACCOUNT}-${REGION}
+export ECR_BASE_URI ?= ${AWS_ACCOUNT}.dkr.ecr.${REGION}.amazonaws.com
+export BUILD_REPOSITORY ?= ${STAGE}-${APP_NAME}-build-service
+export LOAD_REPOSITORY ?= ${STAGE}-${APP_NAME}-load-service
+
 target:
 	$(info ${HELP_MESSAGE})
 	@exit 0
 
-
-deploy: deploy.cfn deploy.ecr
-
-deploy.ecr:
-	$(info [*] Logging into ECR...)
-	@aws ecr get-login-password \
-		--region ${REGION} | docker login \
-			--username AWS \
-			--password-stdin $$(aws sts get-caller-identity --query Account --output text).dkr.ecr.${REGION}.amazonaws.com
-
-#	# @$(info [*] Pushing build service image to ECR...)
-	@docker build -t ${STAGE}-${APP_NAME}-build-service build/ && \
-	docker tag ${STAGE}-${APP_NAME}-build-service:latest $$(aws sts get-caller-identity --query Account --output text).dkr.ecr.${REGION}.amazonaws.com/${STAGE}-${APP_NAME}-build-service:latest && \
-	docker push $$(aws sts get-caller-identity --query Account --output text).dkr.ecr.${REGION}.amazonaws.com/${STAGE}-${APP_NAME}-build-service:latest
-
-#	# @$(info [*] Pushing load service image to ECR...)
-	@docker build -t ${STAGE}-${APP_NAME}-load-service load/ && \
-	docker tag ${STAGE}-${APP_NAME}-load-service:latest $$(aws sts get-caller-identity --query Account --output text).dkr.ecr.${REGION}.amazonaws.com/${STAGE}-${APP_NAME}-load-service:latest && \
-	docker push $$(aws sts get-caller-identity --query Account --output text).dkr.ecr.${REGION}.amazonaws.com/${STAGE}-${APP_NAME}-load-service:latest
-
-deploy.cfn:
-	$(info [*] Deploying...)
-	@bash scripts/deploy.sh ${STAGE} ${APP_NAME} ${REGION} ${NEO4J_USERNAME} ${NEO4J_PASSWORD}
-
-run: ##=> Load an IMGT/HLA release version; make run release=3450 align=False kir=False mem_profile=False limit=1000
-	$(info [*] Starting StepFunctions execution for release $(release))
-
-#	@# TODO: Add validation for positional arguments: release, align, kir, mem_profile, limit
-	@echo "Execution running:"
-	@aws stepfunctions start-execution \
-	 	--state-machine-arn $$(aws ssm get-parameter --name "/${APP_NAME}/${STAGE}/${REGION}/UpdatePipelineArn" | jq -r '.Parameter.Value') \
-	 	--input "{\"params\":{\"environment\":{\"RELEASES\":\"$(release)\",\"ALIGN\":\"False\",\"KIR\":\"False\",\"MEM_PROFILE\":\"False\",\"LIMIT\":\"$(limit)\"}}}" | jq '.executionArn'
-
-delete: ##=> Delete resources
-	$(info [*] Deleting resources...)
-	@bash scripts/delete.sh ${STAGE} ${APP_NAME} ${REGION}
+deploy: check-env ##=> Deploy services
+	$(info [*] Deploying ${APP_NAME} to ${AWS_ACCOUNT}...)
+	$(MAKE) deploy.infrastructure
+	$(MAKE) deploy.database
+	$(MAKE) deploy.pipeline
+	@echo "Finished deploying ${APP_NAME}."
+
+check-env:
+ifndef AWS_PROFILE
+$(error AWS_PROFILE is not set. Please select an AWS profile to use.)
+endif
+ifndef NEO4J_USERNAME
+$(error NEO4J_USERNAME is not set.)
+endif
+ifndef NEO4J_PASSWORD
+$(error NEO4J_PASSWORD is not set.)
+endif
+ifndef GITHUB_PERSONAL_ACCESS_TOKEN
+$(error GITHUB_PERSONAL_ACCESS_TOKEN is not set.)
+endif
+	@echo "Found environment variables"
+
+# Deploy specific stacks
+deploy.infrastructure:
+	$(MAKE) -C gfe-db/infrastructure/ deploy
+
+deploy.database:
+	$(MAKE) -C gfe-db/database/ deploy
+
+deploy.pipeline:
+	$(MAKE) -C gfe-db/pipeline/ deploy
+
+delete: ##=> Delete services
+	$(info [*] Deleting ${APP_NAME} in ${AWS_ACCOUNT}...)
+	$(MAKE) delete.pipeline
+	$(MAKE) delete.database
+	$(MAKE) delete.infrastructure
+	@echo "Finished deleting ${APP_NAME}."
+
+# Delete specific stacks
+delete.infrastructure:
+	$(MAKE) -C gfe-db/infrastructure/ delete
+
+delete.database:
+	$(MAKE) -C gfe-db/database/ delete
+
+delete.pipeline:
+	$(MAKE) -C gfe-db/pipeline/ delete
+
+# run: ##=> Load an IMGT/HLA release version; make run release=3450 align=False kir=False mem_profile=False limit=1000
+# 	$(info [*] Starting StepFunctions execution for release $(release))
+
+# #	@# TODO: Add validation for positional arguments: release, align, kir, mem_profile, limit
+# 	@echo "Execution running:"
+# 	@aws stepfunctions start-execution \
+# 	 	--state-machine-arn $$(aws ssm get-parameter --name "/${APP_NAME}/${STAGE}/${REGION}/UpdatePipelineArn" | jq -r '.Parameter.Value') \
+# 	 	--input "{\"params\":{\"environment\":{\"RELEASES\":\"$(release)\",\"ALIGN\":\"False\",\"KIR\":\"False\",\"MEM_PROFILE\":\"False\",\"LIMIT\":\"$(limit)\"}}}" | jq '.executionArn'
 
 define HELP_MESSAGE
 
 	Environment variables:
 
 	STAGE: "${STAGE}"
 		Description: Feature branch name used as part of stacks name
+
 	APP_NAME: "${APP_NAME}"
 		Description: Stack Name already deployed
+
+	AWS_ACCOUNT: "${AWS_ACCOUNT}":
+		Description: AWS account ID for deployment
+
 	REGION: "${REGION}":
 		Description: AWS region for deployment
-	NEO4J_USERNAME: "${NEO4J_USERNAME}"
-		Description: Neo4j username
-	NEO4J_PASSWORD: "${NEO4J_PASSWORD}"
-		Description: Neo4j password
+
+	DATA_BUCKET_NAME "$${DATA_BUCKET_NAME}"
+		Description: Name of the S3 bucket for data, config and logs
+
+	ECR_BASE_URI: "$${ECR_BASE_URI}"
+		Description: Base URI for AWS Elastic Container Registry
+
+	BUILD_REPOSITORY: "$${BUILD_REPOSITORY}"
+		Description: Name of the ECR repository for the build service
+
+	LOAD_REPOSITORY: "$${LOAD_REPOSITORY}"
+		Description: Name of the ECR repository for the load service
 
 	Common usage: