Merge pull request #1412 from alphagov/ianhowell-gds/add_chat_test_op…

…ensearch_cluster

Configure Integration Opensearch environment to allow snapshots to be imported by Test env.

terraform/deployments/opensearch/README.md

@@ -1,12 +1,12 @@
 ## Chat OpenSearch Snapshots - `register-snapshot-repository.py`
 This document details how the S3 buckets created for the backup process should be registered in each environment. Detailed instructions on how to create index snapshots in Amazon OpenSearch Service can be found [here]. Full instructions on how to access the Amazon OpenSearch Dashboard, along with how to get the login credentials, can be found on this [page].
 
-Registering the S3 buckets as snapshot repositories is a manual one-off process to be carried out in each environment (Integration, Staging and Production). The first step is to log in to the OpenSearch Dashboard and map the AWS IAM Role of the user who will register the repositories. This is followed by running the `register-snapshot-repository.py` script. The backup jobs are run as cronjobs on the EKS cluster. The Production snapshot is created first, which gets imported by Staging and then Integration.
+Registering the S3 buckets as snapshot repositories is a manual one-off process to be carried out in each environment (Test, Integration, Staging and Production). The first step is to log in to the OpenSearch Dashboard and map the AWS IAM Role of the user who will register the repositories. This is followed by running the `register-snapshot-repository.py` script. The backup jobs are run as cronjobs on the EKS cluster. The Production snapshot is created first, which gets imported by Staging, Integration and Test.
 
 ### Commands to run to map the IAM Role in the OpenSearch Dashboard:
 
 ```
-eval $(gds aws govuk-[integration|staging|production]-admin -e -art 8h)
+eval $(gds aws govuk-[test|integration|staging|production]-admin -e -art 8h)
 
 OPENSEARCH_URL=$(aws opensearch describe-domain --domain-name chat-engine | jq -r '.DomainStatus.Endpoints.vpc')
 
@@ -24,7 +24,7 @@ source venv/bin/activate
 
 pip install boto3 requests requests-aws4auth
 
-python register-snapshot-repository.py [integration|staging|production]
+python register-snapshot-repository.py [test|integration|staging|production]
 ```
 
 [here]: https://docs.aws.amazon.com/opensearch-service/latest/developerguide/managedomains-snapshots.html

terraform/deployments/opensearch/main.tf

@@ -186,3 +186,13 @@ resource "aws_route53_record" "service_record" {
   ttl     = 300
   records = [aws_opensearch_domain.opensearch.endpoint]
 }
+
+# This CNAME record is for the Test Opensearch snapshot import K8s cronjob:
+resource "aws_route53_record" "test_service_record" {
+  count   = var.govuk_environment == "integration" ? 1 : 0
+  zone_id = data.terraform_remote_state.infra_root_dns_zones.outputs.internal_root_zone_id
+  name    = "chat-opensearch-test.${var.govuk_environment}.govuk-internal.digital"
+  type    = "CNAME"
+  ttl     = 300
+  records = [var.test_opensearch_url]
+}

terraform/deployments/opensearch/register-snapshot-repository.py

@@ -6,15 +6,15 @@
 in Integration, Staging and Production environments, which are run by EKS as cronjobs.
 
 Instructions for running this script:
-$ eval $(gds aws govuk-[integration|staging|production]-admin -e -art 8h)
+$ eval $(gds aws govuk-[test|integration|staging|production]-admin -e -art 8h)
 $ OPENSEARCH_URL=$(aws opensearch describe-domain --domain-name chat-engine | jq -r '.DomainStatus.Endpoints.vpc')
 $ kubectl relay host/$OPENSEARCH_URL 4443:443
 Open https://localhost:4443/_dashboards in a browser and log in
 Map your AWS Role using instructions in Step 1 of https://docs.aws.amazon.com/opensearch-service/latest/developerguide/managedomains-snapshots.html#managedomains-snapshot-registerdirectory
 $ virtualenv venv
 $ source venv/bin/activate
 $ pip install boto3 requests requests-aws4auth
-$ python register-snapshot-repository.py [integration|staging|production]
+$ python register-snapshot-repository.py [test|integration|staging|production]
 """
 
 import os
@@ -23,6 +23,8 @@
 import requests
 from requests_aws4auth import AWS4Auth
 
+# Uncomment the next line and comment out the following one if this is for the Test Environment:
+# host = "https://search-chat-engine-test-dofkxncldpkjd7huoyakdenpbi.eu-west-1.es.amazonaws.com/"
 host = 'https://localhost:4443/'
 region = 'eu-west-1'
 service = 'es'
@@ -58,7 +60,10 @@ def register_repository(name, role_arn, delete_first=False, read_only=False):
 
 delete_first = 'DELETE_FIRST' in os.environ
 
-if sys.argv[1] == 'integration':
+if sys.argv[1] == 'test':
+    role_arn = 'arn:aws:iam::430354129336:role/govuk-test-chat-opensearch-snapshot-role'
+    register_repository('govuk-production', role_arn, delete_first=delete_first, read_only=True)
+elif sys.argv[1] == 'integration':
     role_arn = 'arn:aws:iam::210287912431:role/govuk-integration-chat-opensearch-snapshot-role'
     register_repository('govuk-integration', role_arn, delete_first=delete_first)
     register_repository('govuk-staging', role_arn, delete_first=delete_first, read_only=True)
@@ -70,4 +75,4 @@ def register_repository(name, role_arn, delete_first=False, read_only=False):
     role_arn = 'arn:aws:iam::172025368201:role/govuk-production-chat-opensearch-snapshot-role'
     register_repository('govuk-production', role_arn, delete_first=delete_first)
 else:
-    print('expected one of [integration|staging|production]')
+    print('expected one of [test|integration|staging|production]')

terraform/deployments/opensearch/s3.tf

@@ -72,10 +72,11 @@ data "aws_iam_policy_document" "opensearch_snapshot_bucket_policy" {
         "172025368201", # Production
         "696911096973", # Staging
         "210287912431", # Integration
+        "430354129336", # Test
       ]
     }
-    # This bucket is only for copying the indices from prod to
-    # staging/integration. Backup snapshot of prod are stored separately, so
+    # This bucket is only for copying the indices from prod to staging,
+    # integration and test. Backup snapshot of prod are stored separately, so
     # the (required) put/delete permissions here don't represent a problem.
     actions = [
       "s3:ListBucket",

terraform/deployments/opensearch/variables.tf

@@ -47,3 +47,8 @@ variable "govuk_environment" {
   type        = string
   description = "Acceptable values are test, integration, staging, production"
 }
+variable "test_opensearch_url" {
+  type        = string
+  description = "The public endpoint for chat-engine-test Opensearch cluster"
+  default     = "chat-opensearch.test.govuk-internal.digital"
+}