38 instrument list and register is not very efficient (#42)

.github/workflows/ci.yml

@@ -0,0 +1,303 @@
+name: continuous-integration
+
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - "v*"
+  pull_request:
+    branches:
+      - "main"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  REGISTRY: ghcr.io
+  POETRY_CACHE_DIR: ~/.cache/pypoetry
+  PIPX_BIN_DIR: /usr/local/bin
+  IMAGE_NAME: ${{ github.repository }}
+  PYTHON_VERSION: "3.12"
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install poetry
+        run: pipx install poetry
+
+      - name: Set up Python ${{ env.PYTHON_VERSION }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+          cache: "poetry"
+
+      - name: Install dependencies
+        run: poetry install
+
+      - name: run ruff
+        run: poetry run ruff check --output-format=github
+
+      - name: Run format
+        run: poetry run ruff format --check
+
+      - name: Run pyright
+        run: poetry run pyright
+
+  security:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install poetry
+        run: pipx install poetry
+
+      - name: Set up Python ${{ env.PYTHON_VERSION }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+          cache: "poetry"
+
+      - name: Install dependencies
+        run: poetry install
+
+      - name: check licenses used by project in pyproject.toml
+        run: poetry run liccheck -s pyproject.toml
+
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@master
+        with:
+          trivy-config: trivy.yaml
+          scan-type: fs
+          scan-ref: "."
+
+  test-compose:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - run: docker compose build
+      - run: docker compose down -v --remove-orphans
+      - run: docker compose up -d
+
+  test-local-backend:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Install poetry
+        run: pipx install poetry
+
+      - name: Set up Python ${{ env.PYTHON_VERSION }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+          cache: "poetry"
+
+      - name: Install dependencies
+        run: poetry install
+
+      - name: Run pytest
+        run: TZ=UTC poetry run coverage run -m pytest
+
+      - name: run coverage report
+        run: poetry run coverage report
+
+      - name: run coverage html
+        run: poetry run coverage html
+
+      - name: Upload code coverage report
+        uses: actions/upload-artifact@v3
+        with:
+          name: codecoverage-${{ github.sha }}
+          path: htmlcov/
+          if-no-files-found: error
+          overwrite: true
+
+      - name: run coverage xml
+        run: poetry run coverage xml
+
+      - name: SonarCloud Scan
+        if: github.actor != 'dependabot[bot]' && !env.ACT
+        uses: SonarSource/sonarcloud-github-action@master
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any
+          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+
+  build:
+    needs: [test-local-backend, test-compose]
+    if: ${{ !github.event.act }}
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+      security-events: write
+      actions: read
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: get commit hash
+        id: get_commit_hash
+        run: |
+          echo "commit_hash=$(git describe --tags)" >> "$GITHUB_OUTPUT"
+
+      - name: Make changes to project to inject commit hash
+        run: |
+          sed -i 's/VERSION: str = .*$/VERSION: str = "${{ steps.get_commit_hash.outputs.commit_hash }}"/g' instrument_registry/core/config.py
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Extract metadata for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} #TODO(berry): fix on git labels multiple tags
+        env:
+          DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,index
+
+      - name: print metadata
+        run: |
+          echo "tags: ${{ steps.meta.outputs.tags }}"
+          echo "labels: ${{ steps.meta.outputs.labels }}"
+          echo "annotations: ${{ steps.meta.outputs.annotations }}"
+          echo "hash: ${{ steps.get_commit_hash.outputs.commit_hash }}"
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ./Dockerfile
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          annotations: ${{ steps.meta.outputs.annotations }}
+          platforms: linux/amd64,linux/arm64,darwin/amd64
+
+      - name: Run Trivy vulnerability scanner sarif
+        uses: aquasecurity/trivy-action@master
+        with:
+          image-ref: ${{ steps.meta.outputs.tags }}
+          scan-type: image
+          exit-code: 0
+          format: "sarif"
+          output: "trivy-results.sarif"
+        env:
+          TRIVY_USERNAME: ${{ github.actor }}
+          TRIVY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Upload Trivy scan results to GitHub Security tab
+        uses: github/codeql-action/upload-sarif@v3
+        with:
+          sarif_file: "trivy-results.sarif"
+
+      - name: Extract metadata for Docker
+        id: meta2
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} #TODO(berry): fix on git labels multiple tags
+          flavor: |
+            latest=false
+        env:
+          DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,index
+
+      - name: Run Trivy SBOM
+        uses: aquasecurity/trivy-action@master
+        with:
+          image-ref: ${{ steps.meta2.outputs.tags }}
+          scan-type: image
+          exit-code: 0
+          format: "cyclonedx"
+          output: "trivy-sbom.json"
+          list-all-pkgs: "true"
+        env:
+          TRIVY_USERNAME: ${{ github.actor }}
+          TRIVY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Run Trivy license scanner
+        uses: aquasecurity/trivy-action@master
+        with:
+          image-ref: ${{ steps.meta2.outputs.tags }}
+          scan-type: image
+          scanners: "license"
+          exit-code: 0
+          output: "trivy-license.json"
+        env:
+          TRIVY_USERNAME: ${{ github.actor }}
+          TRIVY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Upload SBOM & License
+        uses: actions/upload-artifact@v4
+        with:
+          name: sbom-licence-${{ github.sha }}.json
+          path: |
+            trivy-sbom.json
+            trivy-license.json
+          if-no-files-found: error
+          overwrite: true
+
+  deploy:
+    runs-on: ubuntu-latest
+    needs: [build]
+    if: ${{ github.event_name == 'push' && !github.event.act}}
+    permissions:
+      actions: write
+    steps:
+      - name: Extract metadata for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: "" # make empty to get the correct tag
+          flavor: |
+            latest=false
+
+      - name: print metadata
+        run: |
+          echo "tags: ${{ steps.meta.outputs.tags }}"
+
+      - uses: actions/checkout@v4
+
+      - name: Trigger deployment
+        run: |
+          gh workflow run deploy.yml -f image_tag=${{ steps.meta.outputs.tags }} -f environment=production
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+  notifyMattermost:
+    runs-on: ubuntu-latest
+    needs:
+      [
+        lint,
+        security,
+        test-local-backend,
+        test-compose,
+        build,
+      ]
+    if: ${{ always() && contains(needs.*.result, 'failure') }}
+    steps:
+      - uses: mattermost/action-mattermost-notify@master
+        if: github.event_name != 'pull_request' || github.event.pull_request.draft == false
+        with:
+          MATTERMOST_WEBHOOK_URL: ${{ secrets.MM_WEBHOOK_URL }}
+          MATTERMOST_CHANNEL: dev
+          TEXT: |
+            ${{ github.repository }} failed build @here :unamused:
+            :rotating_light: [Pipeline](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) failed :fire:
+          MATTERMOST_USERNAME: ${{ github.triggering_actor }}

.github/workflows/deploy.yml

@@ -5,15 +5,27 @@ on:
     branches:
       - main
 
+env:
+  POETRY_CACHE_DIR: ~/.cache/pypoetry
+  PIPX_BIN_DIR: /usr/local/bin
+
 jobs:
-  build:
+  build: # TODO: REMOVE THIS STEP AFTER AMT HAS BEEN CHANGED TO USE THE API
     runs-on: ubuntu-latest
 
     steps:
     - uses: actions/checkout@v4
-    - uses: actions/setup-python@v5
+    - name: Install poetry
+      run: pipx install poetry
+
+    - name: Set up Python 3.12
+      uses: actions/setup-python@v5
       with:
-        python-version: '3.12'
+        python-version: 3.12
+        cache: "poetry"
+
+    - name: Install dependencies
+      run: poetry install
 
     - name: Configure Git Identity
       run: |
@@ -32,3 +44,49 @@ jobs:
         git push --force origin deployment
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get GHCR package hash
+        id: get_package_hash
+        run: |
+            container_id=$(gh api --paginate -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" /orgs/MinBZK/packages/container/instrument-registry/versions | jq -r '.[] | select(.metadata.container.tags | contains(["main"])) | .name')
+            echo "container_id=$container_id" >> "$GITHUB_OUTPUT"
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: check correct name
+        run: |
+            if [ -z "${{steps.get_package_hash.outputs.container_id}}" ]; then
+              echo "Variable is empty. Failing the workflow."
+              exit 1
+            fi
+
+      - uses: actions/checkout@v4
+        with:
+          repository: 'minbzk/ai-validation-infra'
+          ref: main
+          token: ${{ secrets.GH_PAT }}
+
+      - name: Configure Git
+        run: |
+          git config --global user.name "github-actions[bot]"
+          git config --global user.email "github-actions[bot]@users.noreply.github.com"
+
+
+      - name: Make changes to the file
+        run: |
+          sed -i 's/newTag: .*$/newTag: main@${{ steps.get_package_hash.outputs.container_id }}/g' apps/instrument-registry/sandbox/kustomization.yaml
+          sed -i 's|minbzk.github.io/version: .*$|minbzk.github.io/version: main|g' apps/instrument-registry/sandbox/kustomization.yaml
+          git add apps/instrument-registry/sandbox/kustomization.yaml
+
+      - name: show changes
+        run: git diff --staged
+
+      - name: push changes
+        run: |
+          git commit -m "Update instrument-registry sandbox tag ${{ steps.get_package_hash.outputs.container_id }}"
+          git push --force-with-lease
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_PAT }}