Merge pull request #831 from i-dot-ai/feature/add-django-command-ecs

Add basic django-command Module

Makefile

@@ -229,17 +229,17 @@ tf_set_or_create_workspace:
 
 .PHONY: tf_init
 tf_init: ## Initialise terraform
-	terraform -chdir=./infrastructure/aws/$(instance) init -backend-config=$(TF_BACKEND_CONFIG) ${args} -reconfigure
+	terraform -chdir=./infrastructure/aws/$(instance) init -backend-config=$(TF_BACKEND_CONFIG) ${args}
 
 .PHONY: tf_plan
 tf_plan: ## Plan terraform
 	make tf_set_workspace && \
-	terraform -chdir=./infrastructure/aws/$(instance) plan -var-file=$(CONFIG_DIR)/${env}-input-params.tfvars ${tf_build_args} -target=module.lambda-cleanup
+	terraform -chdir=./infrastructure/aws/$(instance) plan -var-file=$(CONFIG_DIR)/${env}-input-params.tfvars ${tf_build_args}
 
 .PHONY: tf_apply
 tf_apply: ## Apply terraform
 	make tf_set_workspace && \
-	terraform -chdir=./infrastructure/aws/$(instance) apply -var-file=$(CONFIG_DIR)/${env}-input-params.tfvars ${tf_build_args} ${args} -target=module.lambda-cleanup -target=module.rds -target=aws_security_group_rule.lambda_to_rds_egress -target=module.elasticache -target=module.lambda-test -target=module.lambda -target=aws_security_group.service_security_group -target=aws_security_group_rule.lambda_to_443_egress
+	terraform -chdir=./infrastructure/aws/$(instance) apply -var-file=$(CONFIG_DIR)/${env}-input-params.tfvars ${tf_build_args} ${args}
 
 .PHONY: tf_init_universal
 tf_init_universal: ## Initialise terraform

infrastructure/aws/data.tf

@@ -96,6 +96,7 @@ locals {
   reconstructed_worker_secrets = [for k, _ in local.worker_secrets : { name = k, valueFrom = "${aws_secretsmanager_secret.worker-secret.arn}:${k}::" }]
   reconstructed_core_secrets   = [for k, _ in local.core_secrets : { name = k, valueFrom = "${aws_secretsmanager_secret.core-api-secret.arn}:${k}::" }]
   reconstructed_django_secrets = [for k, _ in local.django_app_secrets : { name = k, valueFrom = "${aws_secretsmanager_secret.django-app-secret.arn}:${k}::" }]
+  reconstructed_django_command_secrets = [for k, _ in local.django_app_secrets : { name = k, valueFrom = "${aws_secretsmanager_secret.django-command-secret.arn}:${k}::" }]
 }
 
 data "terraform_remote_state" "vpc" {

infrastructure/aws/ecs.tf

@@ -54,6 +54,13 @@ resource "aws_secretsmanager_secret" "django-app-secret" {
   }
 }
 
+resource "aws_secretsmanager_secret" "django-command-secret" {
+  name = "${local.name}-django-command-secret"
+  tags = {
+    "platform:secret-purpose" = "general"
+  }
+}
+
 resource "aws_secretsmanager_secret" "worker-secret" {
   name = "${local.name}-worker-secret"
   tags = {
@@ -71,6 +78,11 @@ resource "aws_secretsmanager_secret_version" "django-app-json-secret" {
   secret_string = jsonencode(local.django_app_secrets)
 }
 
+resource "aws_secretsmanager_secret_version" "django-command-json-secret" {
+  secret_id     = aws_secretsmanager_secret.django-command-secret.id
+  secret_string = jsonencode(local.django_app_secrets)
+}
+
 resource "aws_secretsmanager_secret_version" "worker-json-secret" {
   secret_id     = aws_secretsmanager_secret.worker-secret.id
   secret_string = jsonencode(local.worker_secrets)
@@ -108,6 +120,32 @@ module "django-app" {
   secrets                      = local.reconstructed_django_secrets
 }
 
+module "django-command" {
+  memory                       = 512
+  cpu                          = 256
+  create_listener              = false
+  create_networking            = false
+  source                       = "../../../i-ai-core-infrastructure//modules/ecs"
+  name                         = "${local.name}-django-command"
+  image_tag                    = var.image_tag
+  command                      = ["venv/bin/django-admin", var.django_command]
+  ecr_repository_uri           = "${var.ecr_repository_uri}/${var.project_name}-django-app"
+  ecs_cluster_id               = module.cluster.ecs_cluster_id
+  ecs_cluster_name             = module.cluster.ecs_cluster_name
+  autoscaling_minimum_target   = 1
+  autoscaling_maximum_target   = 1
+  state_bucket                 = var.state_bucket
+  vpc_id                       = data.terraform_remote_state.vpc.outputs.vpc_id
+  private_subnets              = data.terraform_remote_state.vpc.outputs.private_subnets
+  container_port               = 8091
+  load_balancer_security_group = module.load_balancer.load_balancer_security_group_id
+  aws_lb_arn                   = module.load_balancer.alb_arn
+  host                         = local.django_host
+  environment_variables        = local.django_app_environment_variables
+  secrets                      = local.reconstructed_django_command_secrets
+  http_healthcheck             = false
+  ephemeral_storage            = 30
+}
 
 module "core_api" {
   service_discovery_service_arn = aws_service_discovery_service.service_discovery_service.arn
@@ -179,3 +217,13 @@ resource "aws_security_group_rule" "ecs_ingress_front_to_back" {
   source_security_group_id = module.django-app.ecs_sg_id
   security_group_id        = module.core_api.ecs_sg_id
 }
+
+resource "aws_security_group_rule" "ecs_command_to_core" {
+  type                     = "ingress"
+  description              = "Allow all traffic from the django-command to the core-api"
+  from_port                = 0
+  to_port                  = 0
+  protocol                 = "-1"
+  source_security_group_id = module.django-command.ecs_sg_id
+  security_group_id        = module.core_api.ecs_sg_id
+}

infrastructure/aws/iam.tf

@@ -1,4 +1,3 @@
-
 data "aws_iam_policy_document" "ecs_exec_role_policy" {
   # checkov:skip=CKV_AWS_111:Allow for write access without constraints
   # checkov:skip=CKV_AWS_356:Allow for policies to not have resource limits
@@ -32,6 +31,8 @@ data "aws_iam_policy_document" "ecs_exec_role_policy" {
        "${aws_secretsmanager_secret.worker-secret.arn}:*",
        aws_secretsmanager_secret.django-app-secret.arn,
        "${aws_secretsmanager_secret.django-app-secret.arn}:*",
+       aws_secretsmanager_secret.django-command-secret.arn,
+       "${aws_secretsmanager_secret.django-command-secret.arn}:*",
     ]
   }
 
@@ -56,6 +57,7 @@ resource "aws_iam_role_policy_attachment" "redbox_role_policy" {
       "core-api" = module.core_api.ecs_task_execution_exec_role_name,
       "worker"   = module.worker.ecs_task_execution_exec_role_name,
       "django"   = module.django-app.ecs_task_execution_exec_role_name,
+      "django-command"   = module.django-command.ecs_task_execution_exec_role_name,
     }
   )
   role       = each.value

infrastructure/aws/variables.tf

@@ -24,6 +24,12 @@ variable "developer_ips" {
   description = "List of developer IPs"
 }
 
+variable "django_command" {
+  type        = string
+  default     = "delete_expired_data"
+  description = "Name of Django management to be run. Use with caution"
+}
+
 variable "django_secret_key" {
   type        = string
   description = "cryptographic signature for django app"
@@ -352,4 +358,4 @@ variable "summarisation_chunk_max_tokens" {
   type        = number
   default     = 20000
   description = "Maximum size (in tokens) of chunk used in summarisation"
-}
+}