Merge branch 'development' into hotfix/change-for-merge

.github/workflows/contentful-backup.yml

@@ -0,0 +1,108 @@
+name: Contentful Backup
+on:
+  schedule:
+    - cron: 0 0 * * 1
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: Which Contentful environment to backup
+        required: true
+        type: choice
+        options: [ 'Dev', 'Tst', 'Staging', 'Production' ]
+        default: 'Staging'
+
+jobs:
+  backup_content:
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.environment || 'Staging' }}
+    name: Backup content for ${{ inputs.environment || 'Staging' }}
+    env:
+      az_keyvault_name: ${{ secrets.AZ_ENVIRONMENT }}${{ secrets.DFE_PROJECT_NAME }}-kv
+      az_resource_group_name: ${{ secrets.AZ_ENVIRONMENT }}${{ secrets.DFE_PROJECT_NAME }}
+      az_sql_database_server_name: ${{ secrets.AZ_ENVIRONMENT }}${{ secrets.DFE_PROJECT_NAME }}
+      az_keyvault_contentful_environment: contentful--environment
+      az_keyvault_contentful_space_id: contentful--spaceid
+      az_keyvault_contentful_delivery_token: contentful--deliveryapikey
+      az_keyvault_contentful_backup_storage_key: contentful--backupstoragekey
+      az_keyvault_contentful_management_token: contentful--managementtoken
+      SQL_IP_NAME: export-processor
+      USE_PREVIEW: true
+      SAVE_FILE: true
+      OUTPUT_FILE_DIR: "./output/"
+
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Login with AZ
+        uses: ./.github/actions/azure-login
+        with:
+          az_tenant_id: ${{ secrets.AZ_TENANT_ID }}
+          az_subscription_id: ${{ secrets.AZ_SUBSCRIPTION_ID }}
+          az_client_id: ${{ secrets.AZ_CLIENT_ID }}
+          az_client_secret: ${{ secrets.AZ_CLIENT_SECRET }}
+
+      - name: Get workflow IP address
+        id: whats-my-ip
+        uses: ./.github/actions/whats-my-ip-address
+
+      - name: Add Azure firewall rules
+        uses: ./.github/actions/azure-ip-whitelist
+        with:
+          ip_address: ${{ steps.whats-my-ip.outputs.ip }}
+          verb: "add"
+          az_keyvault_name: ${{ env.az_keyvault_name }}
+          az_ip_name: ${{ env.SQL_IP_NAME }}
+          az_resource_group: ${{ env.az_resource_group_name}}
+          az_sql_database_server_name: ${{ env.az_sql_database_server_name }}
+
+      - name: Get secrets
+        id: get-contentful-export-secrets
+        shell: bash
+        run: |
+          environment=$(az keyvault secret show --name ${{ env.az_keyvault_contentful_environment }} --vault-name ${{ env.az_keyvault_name }} --query value -o tsv)
+          space_id=$(az keyvault secret show --name ${{ env.az_keyvault_contentful_space_id }} --vault-name ${{ env.az_keyvault_name }} --query value -o tsv)
+          delivery_token=$(az keyvault secret show --name ${{ env.az_keyvault_contentful_delivery_token }} --vault-name ${{ env.az_keyvault_name }} --query value -o tsv)
+          backup_storage_key=$(az keyvault secret show --name ${{ env.az_keyvault_contentful_backup_storage_key }} --vault-name ${{ env.az_keyvault_name }} --query value -o tsv)
+          management_token=$(az keyvault secret show --name ${{ env.az_keyvault_contentful_management_token }} --vault-name ${{ env.az_keyvault_name }} --query value -o tsv)
+          echo "::add-mask::$environment"
+          echo "::add-mask::$space_id"
+          echo "::add-mask::$delivery_token"
+          echo "::add-mask::$backup_storage_key"
+          echo "::add-mask::$management_token"
+          echo "ENVIRONMENT=$environment" >> $GITHUB_ENV
+          echo "SPACE_ID=$space_id" >> $GITHUB_ENV
+          echo "DELIVERY_TOKEN=$delivery_token" >> $GITHUB_ENV
+          echo "BACKUP_STORAGE_KEY=$backup_storage_key" >> $GITHUB_ENV
+          echo "MANAGEMENT_TOKEN=$management_token" >> $GITHUB_ENV
+
+      - name: Install contentful-exporter
+        working-directory: ./contentful/export-processor
+        run: npm i
+
+      - name: Export contentful data
+        working-directory: ./contentful/export-processor
+        run: npm run export-all-only
+
+      - name: Upload export to Azure storage container
+        working-directory: ./contentful/export-processor
+        run: |
+          json_files=(output/**/*.json)
+          backup="${json_files[0]}"
+          account_name=$(echo "${az_resource_group_name}content" | sed 's/-//g')
+          az storage blob upload \
+            --account-name "$account_name" \
+            --account-key "$BACKUP_STORAGE_KEY" \
+            --container-name backups-container \
+            --file "$backup"
+
+      - name: Remove Azure Firewall Rules
+        if: always()
+        uses: ./.github/actions/azure-ip-whitelist
+        with:
+          ip_address: ${{ steps.what-is-my-ip.outputs.ip }}
+          verb: "remove"
+          az_keyvault_name: ${{ env.az_keyvault_name }}
+          az_ip_name: ${{ env.SQL_IP_NAME }}
+          az_resource_group: ${{ env.az_resource_group_name}}
+          az_sql_database_server_name: ${{ env.az_sql_database_server_name }}

README.md

@@ -12,7 +12,7 @@ Web application to help schools plan a technology roadmap
 
 ## Running Locally
 
-- The startup project is [./src/Dfe.PlanTech.Web](./src/Dfe.PlanTech.Web)
+- The startup project is [./src/Dfe.PlanTech.Web](./src/Dfe.PlanTech.Web) with setup guidance in the [Readme](./src/Dfe.PlanTech.Web/README.md)
 
 ## Documentation 
 

contentful/content-management/.env.example

@@ -8,3 +8,6 @@ CONTENT_FILE="./some-content.json"
 
 # Skip imports of content models when using importContent
 SKIP_CONTENT_MODEL=true
+
+# Deletes all existing data prior to doing the import
+DELETE_ALL_DATA=false

contentful/content-management/README.md

@@ -4,14 +4,18 @@ Guidance for creating, updating and deleting content programatically from conten
 
 Unfortunately the contentful-migration api does not support deleting entries, so this is something that has to be done
 with the content management api instead. Migrations, where possible, should be done with contentful-migration.
-This can be used as a fallback
+This can be used as a fallback.
+
+The project is currently separate from the content-migrations project because the contentful-migration library doesn't execute changes when the script is run, instead, it outputs a plan for the migration which you can accept or reject (similar to terraform). The contentful-management library does execute changes immediately, so would run during plan creation rather than at the intended stage. To minimise the risk of developer error, these are kept apart.
 
 ## Setup
 
 1. Setup `.env` (copy `.env.example` and setup fields as necessary)
 2. cd into the `content-management` directory
 3. run `npm install`
 
+Note that the `contentful-management` library defaults to the `master` environment if there is a typo in the environment name. This is very undesirable so `validate-environment.js` will fetch all environments from contentful and ensure that `ENVIRONMENT` is one of them.
+
 ## Usage
 
 1. Add your content update script following the convention of `YYYYMMDD-HHMM-description-of-crud-operation.js`
@@ -25,7 +29,9 @@ ____
 
 - importing content can be achieved using the `import-content` script, which will import content from a json file into a specified environment. The json file should be in the format of the `export-processor` export outputs
 - `import-content` uses the `contentful-import` npm package to do the importing
-
+- By default a content import will add to an environment and not remove unrelated data
+  - It will overwrite entries with the same id as one in the import file, and leave other entries alone
+  - To delete all existing data prior to import so that the environment exactly matches the import file, set `DELETE_ALL_DATA` to true in `.env`
 
 ### Usage
 
@@ -39,4 +45,26 @@ Required Environment variables
 `SPACE_ID`
 `MANAGEMENT_TOKEN`
 `ENVIRONMENT` (environmentId default is 'master')
-`SKIP_CONTENT_MODEL` (optional, default is false)
+`SKIP_CONTENT_MODEL` (optional, default is false)
+`DELETE_ALL_DATA` (optional, default is false)
+
+## Deleting all content
+
+You may wish to remove all content from an environment, for example, before importing from a backup. 
+This is an environment variable option in the import process, or it can be run standalone with the `delete-all-content` script, which will clear down the entire environment. 
+
+### Setup
+
+1. Copy `.env.example` and rename to `.env`
+2. Populate the required variables in the file
+3. Run the script via your terminal by running `node delete-all-content.js` to delete _all content types_
+
+_Note: If you want to just delete content for specific content type(s), add a variable called *CONTENT_TYPES_TO_DELETE* in your .env file which should contain a comma separated list of content types to delete_
+
+### Warnings
+
+There is absolutely no confirmation before deletion or anything like that. Make sure you use the right environment variables before running it!
+
+You don't want to accidentally delete everything somewhere else.
+
+Highly recommend backing up your data first!

contentful/content-management/delete-all-content.js

@@ -0,0 +1,21 @@
+require("dotenv/config");
+const getClient = require("./helpers/get-client");
+const validateEnvironment = require("./helpers/validate-environment");
+const deleteContentfulContent = require("./helpers/delete-all-content-and-content-types");
+
+async function deleteAllContentfulData() {
+    const client = await getClient();
+    const contentTypesToDelete = process.env.CONTENT_TYPES_TO_DELETE?.split(",");
+
+    console.log(`Deleting ${process.env.CONTENT_TYPES_TO_DELETE ?? 'all data'} from the following environment: ${process.env.ENVIRONMENT}`)
+
+    try {
+        await deleteContentfulContent({ client, contentTypesToDelete });
+        console.log(`Content deleted successfully for ${process.env.ENVIRONMENT}`);
+    } catch (error) {
+        console.error('Error during deletion:', error);
+        throw error;
+    }
+}
+
+deleteAllContentfulData();

contentful/content-management/helpers/delete-all-content-and-content-types.js

@@ -0,0 +1,65 @@
+/**
+ * Deletes content from Contentful based on the given client and content types.
+ *
+ * @param {Object} options - The options for deleting content.
+ * @param {contentful.ClientAPI} options.client - The Contentful client.
+ * @param {import('./types').ContentfulEnvironmentOptions} options.envOptions - Environment options for the Contentful client.
+ * @param {(string[] | undefined)} options.contentTypes - The array of content types to delete. If undefined, will delete _all_ content
+ * @return {Promise} A promise that resolves when the content is deleted.
+ */
+module.exports = async function deleteContentfulContent({ client, contentTypes }) {
+    if (!client) {
+        throw `Contentful client not provided`;
+    }
+
+    if (!contentTypes || !Array.isArray(contentTypes) || contentTypes.length == 0) {
+        return await deleteContentFromContentful({ client });
+    }
+    else {
+        return await deleteContentByContentTypes({ client, contentTypes });
+    }
+}
+
+async function deleteContentByContentTypes({ client, contentTypes }) {
+    for (const contentType of contentTypes) {
+        console.log(`Deleting content for ${contentType}`);
+        await deleteContentFromContentful({ client, query: { content_type: contentType } });
+    }
+}
+
+async function deleteContentFromContentful({ client, envOptions, query }) {
+    let limit = 100;
+
+    while (true) {
+        const entries = await client.entry.getMany({
+            query: {
+                skip: 0,
+                limit: limit,
+                ...query
+            },
+        });
+
+        if (entries.items.length == 0) {
+            break;
+        }
+
+        console.log(`Deleting ${entries.items.length} items out of ${entries.total}`);
+
+        for (const entry of entries.items) {
+            console.log(entry)
+            await unpublishAndDeleteEntry({ entry, client });
+        }
+    }
+}
+
+async function unpublishAndDeleteEntry({ entry, client }) {
+    if (entry.sys.publishedVersion) {
+        await client.entry.unpublish({
+            entryId: entry.sys.id,
+        });
+    }
+
+    await client.entry.delete({
+        entryId: entry.sys.id
+    });
+}

contentful/content-management/helpers/get-client.js

@@ -1,21 +1,6 @@
 const contentful = require("contentful-management");
+const validateEnvironment = require("./validate-environment");
 
-/**
- * Verifys that the environment specified in .env is a valid contentful environment
- * This is important because if it isn't, the management api fails silently
- * and falls back to using the master environment
- *
- * @param {ClientAPI} client
- */
-async function validateEnvironment(client) {
-    const environments = await client.environment.getMany({
-        spaceId: process.env.SPACE_ID,
-    });
-    const validNames = environments.items.map((env) => env.name);
-    if (!validNames.includes(process.env.ENVIRONMENT)) {
-        throw new Error(`Invalid Contentful environment`);
-    }
-}
 
 module.exports = async function getClient() {
     const client = contentful.createClient(

contentful/content-management/helpers/validate-environment.js

@@ -0,0 +1,16 @@
+/**
+ * Verifies that the environment specified in .env is a valid contentful environment
+ * This is important because if it isn't, the management api fails silently
+ * and falls back to using the master environment
+ *
+ * @param {ClientAPI | null} client
+ */
+module.exports = async function validateEnvironment(client) {
+    const environments = await client.environment.getMany({
+        spaceId: process.env.SPACE_ID,
+    });
+    const validNames = environments.items.map((env) => env.name);
+    if (!validNames.includes(process.env.ENVIRONMENT)) {
+        throw new Error(`Invalid Contentful environment`);
+    }
+}

contentful/content-management/import-content.js

@@ -2,7 +2,8 @@ require('dotenv').config();
 
 const contentfulImport = require('contentful-import');
 const fs = require('fs');
-const validateEnvironment = require("./helpers/validate-environment");
+const getClient = require("./helpers/get-client");
+const deleteContentfulContent = require("./helpers/delete-all-content-and-content-types");
 
 async function importContentfulData() {
   const options = {
@@ -12,13 +13,19 @@ async function importContentfulData() {
     environmentId: process.env.ENVIRONMENT,
 	  skipContentModel: process.env.SKIP_CONTENT_MODEL === 'true' ? true : false
   };
-
-  validateEnvironment()
+  const client = await getClient();
 
   if (!fs.existsSync(options.contentFile)) {
     throw new Error(`File not found: ${options.contentFile}`);
   }
 
+  if (process.env.DELETE_ALL_DATA == 'true') {
+    console.log(`Deleting all existing data from ${options.environmentId}`);
+    await deleteContentfulContent({ client: client });
+  }
+
+  console.log("Starting import with the following options:", options)
+
   try {
     await contentfulImport(options);
     console.log(`Import completed successfully from ${options.contentFile}`);

contentful/content-migrations/README.md

@@ -24,11 +24,6 @@ Currently migrations are run manually, there will likely be a follow on piece to
 4. This will show you a plan for the migration about to happen, type `Y/N` to confirm
 5. Re-enable the webhook
 
-## Errors
-
-In the event of an error that causes the database to be updated with incorrect content,
-you can refresh the database from contentful using the [export-processor](../export-processor/README.md)
-
 ## References
 
 This follows the guidance in the contentful documentation on [scripting migrations](https://www.contentful.com/developers/docs/tutorials/cli/scripting-migrations/)