Merge pull request #926 from DFE-Digital/feat/228964/automate-content…

…ful-backups

feat: Add process for automated contentful backups

.github/workflows/contentful-backup.yml

@@ -0,0 +1,108 @@
+name: Contentful Backup
+on:
+  schedule:
+    - cron: 0 0 * * 1
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: Which Contentful environment to backup
+        required: true
+        type: choice
+        options: [ 'Dev', 'Tst', 'Staging', 'Production' ]
+        default: 'Staging'
+
+jobs:
+  backup_content:
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.environment || 'Staging' }}
+    name: Backup content for ${{ inputs.environment || 'Staging' }}
+    env:
+      az_keyvault_name: ${{ secrets.AZ_ENVIRONMENT }}${{ secrets.DFE_PROJECT_NAME }}-kv
+      az_resource_group_name: ${{ secrets.AZ_ENVIRONMENT }}${{ secrets.DFE_PROJECT_NAME }}
+      az_sql_database_server_name: ${{ secrets.AZ_ENVIRONMENT }}${{ secrets.DFE_PROJECT_NAME }}
+      az_keyvault_contentful_environment: contentful--environment
+      az_keyvault_contentful_space_id: contentful--spaceid
+      az_keyvault_contentful_delivery_token: contentful--deliveryapikey
+      az_keyvault_contentful_backup_storage_key: contentful--backupstoragekey
+      az_keyvault_contentful_management_token: contentful--managementtoken
+      SQL_IP_NAME: export-processor
+      USE_PREVIEW: true
+      SAVE_FILE: true
+      OUTPUT_FILE_DIR: "./output/"
+
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Login with AZ
+        uses: ./.github/actions/azure-login
+        with:
+          az_tenant_id: ${{ secrets.AZ_TENANT_ID }}
+          az_subscription_id: ${{ secrets.AZ_SUBSCRIPTION_ID }}
+          az_client_id: ${{ secrets.AZ_CLIENT_ID }}
+          az_client_secret: ${{ secrets.AZ_CLIENT_SECRET }}
+
+      - name: Get workflow IP address
+        id: whats-my-ip
+        uses: ./.github/actions/whats-my-ip-address
+
+      - name: Add Azure firewall rules
+        uses: ./.github/actions/azure-ip-whitelist
+        with:
+          ip_address: ${{ steps.whats-my-ip.outputs.ip }}
+          verb: "add"
+          az_keyvault_name: ${{ env.az_keyvault_name }}
+          az_ip_name: ${{ env.SQL_IP_NAME }}
+          az_resource_group: ${{ env.az_resource_group_name}}
+          az_sql_database_server_name: ${{ env.az_sql_database_server_name }}
+
+      - name: Get secrets
+        id: get-contentful-export-secrets
+        shell: bash
+        run: |
+          environment=$(az keyvault secret show --name ${{ env.az_keyvault_contentful_environment }} --vault-name ${{ env.az_keyvault_name }} --query value -o tsv)
+          space_id=$(az keyvault secret show --name ${{ env.az_keyvault_contentful_space_id }} --vault-name ${{ env.az_keyvault_name }} --query value -o tsv)
+          delivery_token=$(az keyvault secret show --name ${{ env.az_keyvault_contentful_delivery_token }} --vault-name ${{ env.az_keyvault_name }} --query value -o tsv)
+          backup_storage_key=$(az keyvault secret show --name ${{ env.az_keyvault_contentful_backup_storage_key }} --vault-name ${{ env.az_keyvault_name }} --query value -o tsv)
+          management_token=$(az keyvault secret show --name ${{ env.az_keyvault_contentful_management_token }} --vault-name ${{ env.az_keyvault_name }} --query value -o tsv)
+          echo "::add-mask::$environment"
+          echo "::add-mask::$space_id"
+          echo "::add-mask::$delivery_token"
+          echo "::add-mask::$backup_storage_key"
+          echo "::add-mask::$management_token"
+          echo "ENVIRONMENT=$environment" >> $GITHUB_ENV
+          echo "SPACE_ID=$space_id" >> $GITHUB_ENV
+          echo "DELIVERY_TOKEN=$delivery_token" >> $GITHUB_ENV
+          echo "BACKUP_STORAGE_KEY=$backup_storage_key" >> $GITHUB_ENV
+          echo "MANAGEMENT_TOKEN=$management_token" >> $GITHUB_ENV
+
+      - name: Install contentful-exporter
+        working-directory: ./contentful/export-processor
+        run: npm i
+
+      - name: Export contentful data
+        working-directory: ./contentful/export-processor
+        run: npm run export-all-only
+
+      - name: Upload export to Azure storage container
+        working-directory: ./contentful/export-processor
+        run: |
+          json_files=(output/**/*.json)
+          backup="${json_files[0]}"
+          account_name=$(echo "${az_resource_group_name}content" | sed 's/-//g')
+          az storage blob upload \
+            --account-name "$account_name" \
+            --account-key "$BACKUP_STORAGE_KEY" \
+            --container-name backups-container \
+            --file "$backup"
+
+      - name: Remove Azure Firewall Rules
+        if: always()
+        uses: ./.github/actions/azure-ip-whitelist
+        with:
+          ip_address: ${{ steps.what-is-my-ip.outputs.ip }}
+          verb: "remove"
+          az_keyvault_name: ${{ env.az_keyvault_name }}
+          az_ip_name: ${{ env.SQL_IP_NAME }}
+          az_resource_group: ${{ env.az_resource_group_name}}
+          az_sql_database_server_name: ${{ env.az_sql_database_server_name }}

contentful/content-management/.env.example

@@ -8,3 +8,6 @@ CONTENT_FILE="./some-content.json"
 
 # Skip imports of content models when using importContent
 SKIP_CONTENT_MODEL=true
+
+# Deletes all existing data prior to doing the import
+DELETE_ALL_DATA=false

contentful/content-management/README.md

@@ -25,7 +25,9 @@ ____
 
 - importing content can be achieved using the `import-content` script, which will import content from a json file into a specified environment. The json file should be in the format of the `export-processor` export outputs
 - `import-content` uses the `contentful-import` npm package to do the importing
-
+- By default a content import will add to an environment and not remove unrelated data
+  - It will overwrite entries with the same id as one in the import file, and leave other entries alone
+  - To delete all existing data prior to import so that the environment exactly matches the import file, set `DELETE_ALL_DATA` to true in `.env`
 
 ### Usage
 
@@ -39,4 +41,26 @@ Required Environment variables
 `SPACE_ID`
 `MANAGEMENT_TOKEN`
 `ENVIRONMENT` (environmentId default is 'master')
-`SKIP_CONTENT_MODEL` (optional, default is false)
+`SKIP_CONTENT_MODEL` (optional, default is false)
+`DELETE_ALL_DATA` (optional, default is false)
+
+## Deleting all content
+
+You may wish to remove all content from an environment, for example, before importing from a backup. 
+This is an environment variable option in the import process, or it can be run standalone with the `delete-all-content` script, which will clear down the entire environment. 
+
+### Setup
+
+1. Copy `.env.example` and rename to `.env`
+2. Populate the required variables in the file
+3. Run the script via your terminal by running `node delete-all-content.js` to delete _all content types_
+
+_Note: If you want to just delete content for specific content type(s), add a variable called *CONTENT_TYPES_TO_DELETE* in your .env file which should contain a comma separated list of content types to delete_
+
+### Warnings
+
+There is absolutely no confirmation before deletion or anything like that. Make sure you use the right environment variables before running it!
+
+You don't want to accidentally delete everything somewhere else.
+
+Highly recommend backing up your data first!

contentful/content-management/delete-all-content.js

@@ -0,0 +1,21 @@
+require("dotenv/config");
+const getClient = require("./helpers/get-client");
+const validateEnvironment = require("./helpers/validate-environment");
+const deleteContentfulContent = require("./helpers/delete-all-content-and-content-types");
+
+async function deleteAllContentfulData() {
+    const client = await getClient();
+    const contentTypesToDelete = process.env.CONTENT_TYPES_TO_DELETE?.split(",");
+
+    console.log(`Deleting ${process.env.CONTENT_TYPES_TO_DELETE ?? 'all data'} from the following environment: ${process.env.ENVIRONMENT}`)
+
+    try {
+        await deleteContentfulContent({ client, contentTypesToDelete });
+        console.log(`Content deleted successfully for ${process.env.ENVIRONMENT}`);
+    } catch (error) {
+        console.error('Error during deletion:', error);
+        throw error;
+    }
+}
+
+deleteAllContentfulData();

contentful/content-management/helpers/delete-all-content-and-content-types.js

@@ -0,0 +1,65 @@
+/**
+ * Deletes content from Contentful based on the given client and content types.
+ *
+ * @param {Object} options - The options for deleting content.
+ * @param {contentful.ClientAPI} options.client - The Contentful client.
+ * @param {import('./types').ContentfulEnvironmentOptions} options.envOptions - Environment options for the Contentful client.
+ * @param {(string[] | undefined)} options.contentTypes - The array of content types to delete. If undefined, will delete _all_ content
+ * @return {Promise} A promise that resolves when the content is deleted.
+ */
+module.exports = async function deleteContentfulContent({ client, contentTypes }) {
+    if (!client) {
+        throw `Contentful client not provided`;
+    }
+
+    if (!contentTypes || !Array.isArray(contentTypes) || contentTypes.length == 0) {
+        return await deleteContentFromContentful({ client });
+    }
+    else {
+        return await deleteContentByContentTypes({ client, contentTypes });
+    }
+}
+
+async function deleteContentByContentTypes({ client, contentTypes }) {
+    for (const contentType of contentTypes) {
+        console.log(`Deleting content for ${contentType}`);
+        await deleteContentFromContentful({ client, query: { content_type: contentType } });
+    }
+}
+
+async function deleteContentFromContentful({ client, envOptions, query }) {
+    let limit = 100;
+
+    while (true) {
+        const entries = await client.entry.getMany({
+            query: {
+                skip: 0,
+                limit: limit,
+                ...query
+            },
+        });
+
+        if (entries.items.length == 0) {
+            break;
+        }
+
+        console.log(`Deleting ${entries.items.length} items out of ${entries.total}`);
+
+        for (const entry of entries.items) {
+            console.log(entry)
+            await unpublishAndDeleteEntry({ entry, client });
+        }
+    }
+}
+
+async function unpublishAndDeleteEntry({ entry, client }) {
+    if (entry.sys.publishedVersion) {
+        await client.entry.unpublish({
+            entryId: entry.sys.id,
+        });
+    }
+
+    await client.entry.delete({
+        entryId: entry.sys.id
+    });
+}

contentful/content-management/helpers/get-client.js

@@ -1,21 +1,6 @@
 const contentful = require("contentful-management");
+const validateEnvironment = require("./validate-environment");
 
-/**
- * Verifys that the environment specified in .env is a valid contentful environment
- * This is important because if it isn't, the management api fails silently
- * and falls back to using the master environment
- *
- * @param {ClientAPI} client
- */
-async function validateEnvironment(client) {
-    const environments = await client.environment.getMany({
-        spaceId: process.env.SPACE_ID,
-    });
-    const validNames = environments.items.map((env) => env.name);
-    if (!validNames.includes(process.env.ENVIRONMENT)) {
-        throw new Error(`Invalid Contentful environment`);
-    }
-}
 
 module.exports = async function getClient() {
     const client = contentful.createClient(

contentful/content-management/helpers/validate-environment.js

@@ -0,0 +1,16 @@
+/**
+ * Verifies that the environment specified in .env is a valid contentful environment
+ * This is important because if it isn't, the management api fails silently
+ * and falls back to using the master environment
+ *
+ * @param {ClientAPI | null} client
+ */
+module.exports = async function validateEnvironment(client) {
+    const environments = await client.environment.getMany({
+        spaceId: process.env.SPACE_ID,
+    });
+    const validNames = environments.items.map((env) => env.name);
+    if (!validNames.includes(process.env.ENVIRONMENT)) {
+        throw new Error(`Invalid Contentful environment`);
+    }
+}

contentful/content-management/import-content.js

@@ -2,7 +2,8 @@ require('dotenv').config();
 
 const contentfulImport = require('contentful-import');
 const fs = require('fs');
-const validateEnvironment = require("./helpers/validate-environment");
+const getClient = require("./helpers/get-client");
+const deleteContentfulContent = require("./helpers/delete-all-content-and-content-types");
 
 async function importContentfulData() {
   const options = {
@@ -12,13 +13,19 @@ async function importContentfulData() {
     environmentId: process.env.ENVIRONMENT,
 	  skipContentModel: process.env.SKIP_CONTENT_MODEL === 'true' ? true : false
   };
-
-  validateEnvironment()
+  const client = await getClient();
 
   if (!fs.existsSync(options.contentFile)) {
     throw new Error(`File not found: ${options.contentFile}`);
   }
 
+  if (process.env.DELETE_ALL_DATA == 'true') {
+    console.log(`Deleting all existing data from ${options.environmentId}`);
+    await deleteContentfulContent({ client: client });
+  }
+
+  console.log("Starting import with the following options:", options)
+
   try {
     await contentfulImport(options);
     console.log(`Import completed successfully from ${options.contentFile}`);

terraform/container-app/contentful.tf

@@ -19,3 +19,43 @@ resource "null_resource" "upsert_contentful_webhook" {
     command = local.contentful_webhook_shell_command
   }
 }
+
+resource "azurerm_storage_account" "contentful_backup_storage" {
+  name                     = replace("${local.resource_prefix}content", "-", "")
+  resource_group_name      = local.resource_prefix
+  location                 = local.azure_location
+  account_tier             = "Standard"
+  account_replication_type = "LRS"
+  tags                     = local.tags
+
+  blob_properties {
+    delete_retention_policy {
+      days = 30
+    }
+  }
+}
+
+resource "azurerm_storage_container" "backups_container" {
+  name                  = "backups-container"
+  storage_account_name  = azurerm_storage_account.contentful_backup_storage.name
+  container_access_type = "private"
+}
+
+resource "azurerm_storage_management_policy" "lifecycle_policy" {
+  storage_account_id = azurerm_storage_account.contentful_backup_storage.id
+
+  rule {
+    name    = "delete_after_30_days"
+    enabled = true
+
+    filters {
+      blob_types = ["blockBlob"]
+    }
+
+    actions {
+      base_blob {
+        delete_after_days_since_creation_greater_than = 30
+      }
+    }
+  }
+}

terraform/container-app/terraform-configuration.md

@@ -80,8 +80,11 @@ We use two external modules to create the majority of the resources required:
 | [azurerm_servicebus_namespace.service_bus](https://registry.terraform.io/providers/hashicorp/azurerm/4.4.0/docs/resources/servicebus_namespace) | resource |
 | [azurerm_servicebus_queue.contentful_queue](https://registry.terraform.io/providers/hashicorp/azurerm/4.4.0/docs/resources/servicebus_queue) | resource |
 | [azurerm_servicebus_queue_authorization_rule.azurefunction](https://registry.terraform.io/providers/hashicorp/azurerm/4.4.0/docs/resources/servicebus_queue_authorization_rule) | resource |
+| [azurerm_storage_account.contentful_backup_storage](https://registry.terraform.io/providers/hashicorp/azurerm/4.4.0/docs/resources/storage_account) | resource |
 | [azurerm_storage_account.costing_storage](https://registry.terraform.io/providers/hashicorp/azurerm/4.4.0/docs/resources/storage_account) | resource |
+| [azurerm_storage_container.backups_container](https://registry.terraform.io/providers/hashicorp/azurerm/4.4.0/docs/resources/storage_container) | resource |
 | [azurerm_storage_container.blobforcost](https://registry.terraform.io/providers/hashicorp/azurerm/4.4.0/docs/resources/storage_container) | resource |
+| [azurerm_storage_management_policy.lifecycle_policy](https://registry.terraform.io/providers/hashicorp/azurerm/4.4.0/docs/resources/storage_management_policy) | resource |
 | [azurerm_subnet.keyvault](https://registry.terraform.io/providers/hashicorp/azurerm/4.4.0/docs/resources/subnet) | resource |
 | [azurerm_subnet_route_table_association.keyvault](https://registry.terraform.io/providers/hashicorp/azurerm/4.4.0/docs/resources/subnet_route_table_association) | resource |
 | [azurerm_user_assigned_identity.user_assigned_identity](https://registry.terraform.io/providers/hashicorp/azurerm/4.4.0/docs/resources/user_assigned_identity) | resource |