feat: 5950 build logstash container image pipeline

.github/workflows/cd.yaml

@@ -45,6 +45,7 @@ jobs:
       should-build-and-push-cli-docker: ${{ needs.release-please.outputs.release_created || steps.changed-cli-docker-files.outputs.any_modified == 'true' || steps.changed-api-files.outputs.any_modified == 'true' || null }}
       should-build-and-push-selfserve-docker: ${{ needs.release-please.outputs.release_created || steps.changed-selfserve-docker-files.outputs.any_modified == 'true' || steps.changed-selfserve-files.outputs.any_modified == 'true' || null }}
       should-build-and-push-internal-docker: ${{ needs.release-please.outputs.release_created || steps.changed-internal-docker-files.outputs.any_modified == 'true' || steps.changed-internal-files.outputs.any_modified == 'true' || null }}
+      should-build-and-push-search-docker: ${{ needs.release-please.outputs.release_created || steps.changed-search-docker-files.outputs.any_modified == 'true' || steps.changed-search-files.outputs.any_modified == 'true' || null }}
       # Terraform account
       should-apply-account-terraform: ${{ needs.release-please.outputs.release_created || steps.changed-accounts-terraform-files.outputs.any_modified == 'true' || null }}
       # Terraform environment
@@ -93,6 +94,11 @@ jobs:
         with:
           files: |
             infra/docker/internal/**
+      - uses: tj-actions/changed-files@v45
+        id: changed-seaerch-docker-files
+        with:
+          files: |
+            infra/docker/seaerch/**
       - uses: tj-actions/changed-files@v45
         id: changed-accounts-terraform-files
         with:
@@ -144,6 +150,7 @@ jobs:
       selfserve: ${{ steps.selfserve-version.outputs.version }}
       internal: ${{ steps.internal-version.outputs.version }}
       assets: ${{ steps.assets-version.outputs.version }}
+      search: ${{ steps.search-version.outputs.version }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -168,6 +175,10 @@ jobs:
         uses: dvsa/.github/.github/actions/get-vol-app-version@v5.0.3
         with:
           project-path: app/cdn
+      - id: search-version
+        uses: dvsa/.github/.github/actions/get-vol-app-version@v5.0.3
+        with:
+          project-path: infra/docker/search
       - name: Add to summary
         run: |
           echo "#### App versions:" >> $GITHUB_STEP_SUMMARY
@@ -176,6 +187,7 @@ jobs:
           echo "**Selfserve**: \`${{ steps.selfserve-version.outputs.version }}\`" >> $GITHUB_STEP_SUMMARY
           echo "**Internal**: \`${{ steps.internal-version.outputs.version }}\`" >> $GITHUB_STEP_SUMMARY
           echo "**Assets**: \`${{ steps.assets-version.outputs.version }}\`" >> $GITHUB_STEP_SUMMARY
+          echo "**Search**: \`${{ steps.search-version.outputs.version }}\`" >> $GITHUB_STEP_SUMMARY
 
   cdn-nonprod:
     name: CDN
@@ -239,11 +251,13 @@ jobs:
           - cli
           - selfserve
           - internal
+          - search
         exclude:
           - project: ${{ needs.orchestrator.outputs.should-build-and-push-api-docker && 'ignored' || 'api' }}
           - project: ${{ needs.orchestrator.outputs.should-build-and-push-cli-docker && 'ignored' || 'cli' }}
           - project: ${{ needs.orchestrator.outputs.should-build-and-push-selfserve-docker && 'ignored' || 'selfserve' }}
           - project: ${{ needs.orchestrator.outputs.should-build-and-push-internal-docker && 'ignored' || 'internal' }}
+          - project: ${{ needs.orchestrator.outputs.should-build-and-push-search-docker && 'ignored' || 'search' }}
     uses: ./.github/workflows/docker.yaml
     with:
       project: ${{ matrix.project }}

.github/workflows/ci.yaml

@@ -36,7 +36,7 @@ jobs:
       # Docs
       should-build-docs: ${{ steps.changed-website-files.outputs.any_modified == 'true' || null }}
       # App
-      should-build-app: ${{ steps.changed-app-files.outputs.any_modified == 'true' || steps.changed-docker-files.outputs.any_modified == 'true' || null }}
+      should-build-app: ${{ steps.changed-app-files.outputs.any_modified == 'true' || null }}
       should-build-api: ${{ contains(steps.changed-app-files.outputs.all_modified_files, 'app/api') || null }}
       should-build-selfserve: ${{ contains(steps.changed-app-files.outputs.all_modified_files, 'app/selfserve') || null }}
       should-build-internal: ${{ contains(steps.changed-app-files.outputs.all_modified_files, 'app/internal') || null }}
@@ -48,6 +48,7 @@ jobs:
       should-build-cli-docker: ${{ contains(steps.changed-docker-files.outputs.all_modified_files, 'infra/docker/cli') || null }}
       should-build-selfserve-docker: ${{ contains(steps.changed-docker-files.outputs.all_modified_files, 'infra/docker/selfserve') || null }}
       should-build-internal-docker: ${{ contains(steps.changed-docker-files.outputs.all_modified_files, 'infra/docker/internal') || null }}
+      should-build-search-docker: ${{ contains(steps.changed-docker-files.outputs.all_modified_files, 'infra/docker/search') || null }}
       # Terraform accounts
       should-plan-terraform-accounts: ${{ steps.changed-accounts-terraform-files.outputs.any_modified == 'true' || null }}
       should-plan-nonprod-account-terraform: ${{ contains(steps.changed-accounts-terraform-files.outputs.all_modified_files, 'infra/terraform/modules') || contains(steps.changed-accounts-terraform-files.outputs.all_modified_files, 'infra/terraform/accounts/nonprod') || null }}
@@ -142,6 +143,7 @@ jobs:
       selfserve: ${{ steps.selfserve-version.outputs.version }}
       internal: ${{ steps.internal-version.outputs.version }}
       assets: ${{ steps.assets-version.outputs.version }}
+      search: ${{ steps.search-version.outputs.version }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -167,13 +169,18 @@ jobs:
         uses: dvsa/.github/.github/actions/get-vol-app-version@v5.0.3
         with:
           project-path: app/cdn
+      - id: search-version
+        uses: dvsa/.github/.github/actions/get-vol-app-version@v5.0.3
+        with:
+          project-path: infra/docker/search
       - name: Add to summary
         run: |
           echo "#### App versions:" >> $GITHUB_STEP_SUMMARY
           echo "**API**: \`${{ steps.api-version.outputs.version }}\`" >> $GITHUB_STEP_SUMMARY
           echo "**Selfserve**: \`${{ steps.selfserve-version.outputs.version }}\`" >> $GITHUB_STEP_SUMMARY
           echo "**Internal**: \`${{ steps.internal-version.outputs.version }}\`" >> $GITHUB_STEP_SUMMARY
           echo "**Assets**: \`${{ steps.assets-version.outputs.version }}\`" >> $GITHUB_STEP_SUMMARY
+          echo "**Search**: \`${{ steps.search-version.outputs.version }}\`" >> $GITHUB_STEP_SUMMARY
 
   cdn:
     name: CDN
@@ -193,7 +200,11 @@ jobs:
 
   app:
     name: App
-    if: ${{ needs.orchestrator.outputs.should-build-app || needs.orchestrator.outputs.should-build-docker }}
+    if: >
+      needs.orchestrator.outputs.should-build-app ||
+      needs.orchestrator.outputs.should-build-api-docker ||
+      needs.orchestrator.outputs.should-build-selfserve-docker ||
+      needs.orchestrator.outputs.should-build-internal-docker
     concurrency:
       group: app-${{ matrix.project }}-${{ needs.get-version.outputs[matrix.project] }}
     needs:
@@ -221,13 +232,27 @@ jobs:
 
   docker:
     name: Docker
-    if: ${{ always() && !cancelled() && !failure() && needs.orchestrator.outputs.should-build-docker }}
+    if: >
+      always() && 
+      !cancelled() && 
+      needs.orchestrator.outputs.should-build-docker && 
+      needs.security-app.result != 'failure'  && 
+      needs.security-terraform.result != 'failure'  && 
+      needs.orchestrator.result != 'failure'  && 
+      needs.docs.result != 'failure'  && 
+      needs.cdn.result != 'failure'  && 
+      needs.app.result != 'failure'  && 
+      needs.get-version.result != 'failure'
     concurrency:
       group: docker-${{ matrix.project }}-${{ needs.get-version.outputs[matrix.project] }}
     needs:
+      - security-app
+      - security-terraform
       - orchestrator
-      - get-version
+      - docs
+      - cdn
       - app
+      - get-version
     strategy:
       fail-fast: false
       matrix:
@@ -236,11 +261,13 @@ jobs:
           - cli
           - selfserve
           - internal
+          - search
         exclude:
           - project: ${{ needs.orchestrator.outputs.should-build-api-docker && 'ignored' || 'api' }}
           - project: ${{ needs.orchestrator.outputs.should-build-cli-docker && 'ignored' || 'cli' }}
           - project: ${{ needs.orchestrator.outputs.should-build-selfserve-docker && 'ignored' || 'selfserve' }}
           - project: ${{ needs.orchestrator.outputs.should-build-internal-docker && 'ignored' || 'internal' }}
+          - project: ${{ needs.orchestrator.outputs.should-build-search-docker && 'ignored' || 'search' }}
     uses: ./.github/workflows/docker.yaml
     with:
       project: ${{ matrix.project }}
@@ -328,6 +355,7 @@ jobs:
       selfserve-image-tag: ${{ needs.get-version.outputs.selfserve }}
       internal-image-tag: ${{ needs.get-version.outputs.internal }}
       assets-version: ${{ needs.get-version.outputs.assets }}
+      search-image-tag: ${{ needs.get-version.outputs.search }}
     permissions:
       contents: read
       id-token: write

.github/workflows/deploy-environment.yaml

@@ -32,6 +32,10 @@ on:
         description: "Internal image tag"
         type: string
         required: true
+      search-image-tag:
+        description: "Search image tag"
+        type: string
+        required: true
       assets-version:
         description: "Assets version"
         type: string
@@ -70,6 +74,9 @@ on:
       internal-image-tag:
         type: string
         required: true
+      search-image-tag:
+        type: string
+        required: true
       assets-version:
         type: string
         required: true
@@ -101,6 +108,9 @@ on:
       previous_internal_image_tag:
         description: "Previous Internal image tag"
         value: ${{ jobs.deploy.outputs.previous_internal_image_tag }}
+      previous_search_image_tag:
+        description: "Previous Search image tag"
+        value: ${{ jobs.deploy.outputs.previous_search_image_tag }}
       previous_assets_version:
         description: "Previous Assets version"
         value: ${{ jobs.deploy.outputs.previous_assets_version }}
@@ -128,6 +138,7 @@ jobs:
       previous_cli_image_tag: ${{ steps.get_current_versions.outputs.cli_image_tag }}
       previous_selfserve_image_tag: ${{ steps.get_current_versions.outputs.selfserve_image_tag }}
       previous_internal_image_tag: ${{ steps.get_current_versions.outputs.internal_image_tag }}
+      previous_search_image_tag: ${{ steps.get_current_versions.outputs.search_image_tag }}
       previous_assets_version: ${{ steps.get_current_versions.outputs.assets_version }}
     env:
       WORKING_DIR: infra/terraform/environments/${{ inputs.environment }}
@@ -183,6 +194,7 @@ jobs:
           TF_VAR_cli_image_tag: ${{ inputs.cli-image-tag }}
           TF_VAR_selfserve_image_tag: ${{ inputs.selfserve-image-tag }}
           TF_VAR_internal_image_tag: ${{ inputs.internal-image-tag }}
+          TF_VAR_search_image_tag: ${{ inputs.search-image-tag }}
           TF_VAR_assets_version: ${{ inputs.assets-version }}
         run: terraform plan -parallelism=80 ${{ inputs.destroy && '-destroy ' || '' }} -no-color -input=false -out=tfplan ${{ inputs.terraform-args || '' }}
 
@@ -254,6 +266,7 @@ jobs:
             **CLI version:** ${{ inputs.cli-image-tag }}
             **Selfserve version:** ${{ inputs.selfserve-image-tag }}
             **Internal version:** ${{ inputs.internal-image-tag }}
+            **search version:** ${{ inputs.search-image-tag }}
 
             ### Plan summary
 
@@ -295,6 +308,7 @@ jobs:
           TF_VAR_cli_image_tag: ${{ inputs.cli-image-tag }}
           TF_VAR_selfserve_image_tag: ${{ inputs.selfserve-image-tag }}
           TF_VAR_internal_image_tag: ${{ inputs.internal-image-tag }}
+          TF_VAR_search_image_tag: ${{ inputs.search-image-tag }}
           TF_VAR_assets_version: ${{ inputs.assets-version }}
         run: terraform apply -parallelism=80 ${{ inputs.destroy && '-destroy ' || '' }} -no-color -input=false -auto-approve ${{ inputs.terraform-args || '' }}
 
@@ -312,6 +326,7 @@ jobs:
           echo "- CLI: ${{ inputs.cli-image-tag }}" >> $GITHUB_STEP_SUMMARY
           echo "- Selfserve: ${{ inputs.selfserve-image-tag }}" >> $GITHUB_STEP_SUMMARY
           echo "- Internal: ${{ inputs.internal-image-tag }}" >> $GITHUB_STEP_SUMMARY
+          echo "- Search: ${{ inputs.search-image-tag }}" >> $GITHUB_STEP_SUMMARY
           echo "- Assets: ${{ inputs.assets-version }}" >> $GITHUB_STEP_SUMMARY
 
       - name: Set outputs

.github/workflows/docker.yaml

@@ -57,6 +57,7 @@ jobs:
           fetch-depth: ${{ !inputs.ref && 1 || 0 }}
 
       - uses: actions/download-artifact@v4
+        if: ${{ inputs.app-artefact-name != 'search' }}
         with:
           name: ${{ inputs.app-artefact-name }}
           path: infra/docker/${{ inputs.project }}

infra/docker/search/Dockerfile

@@ -0,0 +1,13 @@
+FROM ghcr.io/dvsa/dvsa-docker-images/logstash/8.16.0/batch:0 AS search
+
+FROM search AS production
+
+# Delete default configuration
+RUN rm -f /usr/share/logstash/pipeline/logstash.conf
+
+COPY config/ /usr/share/logstash/config/
+
+COPY entrypoint.sh /usr/share/logstash/entrypoint.sh
+COPY build.sh /usr/share/logstash/build.sh
+
+CMD ["/usr/share/logstash/entrypoint.sh"]