Merge pull request #1747 from stevenc987/main

23714-GCP: Build Notebook Report Job

.github/workflows/notebook-report-cd.yml

@@ -1,4 +1,4 @@
-name: Namex Notebook Report Job CD
+name: Notebook Report CD
 
 on:
   push:
@@ -8,96 +8,23 @@ on:
       - "jobs/notebook-report/**"
   workflow_dispatch:
     inputs:
-      environment:
-        description: "Environment (dev/test/prod)"
+      target:
+        description: "Deploy To"
         required: true
-        default: "dev"
-
-defaults:
-  run:
-    shell: bash
-    working-directory: ./jobs/notebook-report
-
-env:
-  APP_NAME: "notebook-report"
-  TAG_NAME: "dev"
+        type: choice
+        options:
+          - dev
+          - test
+          - sandbox
+          - prod
 
 jobs:
-  notebook-report-cd-by-push:
-    runs-on: ubuntu-20.04
-
-    if: github.event_name == 'push' && github.repository == 'bcgov/namex'
-    environment:
-      name: "dev"
-
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Login Openshift
-        shell: bash
-        run: |
-          oc login --server=${{secrets.OPENSHIFT4_LOGIN_REGISTRY}} --token=${{secrets.OPENSHIFT4_SA_TOKEN}}
-
-      - name: CD Flow
-        shell: bash
-        env:
-          OPS_REPOSITORY: ${{ secrets.OPS_REPOSITORY }}
-          OPENSHIFT_DOCKER_REGISTRY: ${{ secrets.OPENSHIFT4_DOCKER_REGISTRY }}
-          OPENSHIFT_SA_NAME: ${{ secrets.OPENSHIFT4_SA_NAME }}
-          OPENSHIFT_SA_TOKEN: ${{ secrets.OPENSHIFT4_SA_TOKEN }}
-          OPENSHIFT_REPOSITORY: ${{ secrets.OPENSHIFT4_REPOSITORY }}
-          TAG_NAME: ${{ env.TAG_NAME }}
-        run: |
-          make cd
-
-      - name: Rocket.Chat Notification
-        uses: RocketChat/Rocket.Chat.GitHub.Action.Notification@master
-        if: failure()
-        with:
-          type: ${{ job.status }}
-          job_name: "*Notebook Report Job Built and Deployed to ${{env.TAG_NAME}}*"
-          channel: "#registries-bot"
-          url: ${{ secrets.ROCKETCHAT_WEBHOOK }}
-          commit: true
-          token: ${{ secrets.GITHUB_TOKEN }}
-
-  notebook-report-cd-by-dispatch:
-    runs-on: ubuntu-20.04
-
-    if: github.event_name == 'workflow_dispatch' && github.repository == 'bcgov/namex'
-    environment:
-      name: "${{ github.event.inputs.environment }}"
-
-    steps:
-      - uses: actions/checkout@v3
-      - name: Set env by input
-        run: |
-          echo "TAG_NAME=${{ github.event.inputs.environment }}" >> $GITHUB_ENV
-
-      - name: Login Openshift
-        shell: bash
-        run: |
-          oc login --server=${{secrets.OPENSHIFT4_LOGIN_REGISTRY}} --token=${{secrets.OPENSHIFT4_SA_TOKEN}}
-
-      - name: CD Flow
-        shell: bash
-        env:
-          OPS_REPOSITORY: ${{ secrets.OPS_REPOSITORY }}
-          OPENSHIFT_DOCKER_REGISTRY: ${{ secrets.OPENSHIFT4_DOCKER_REGISTRY }}
-          OPENSHIFT_SA_NAME: ${{ secrets.OPENSHIFT4_SA_NAME }}
-          OPENSHIFT_SA_TOKEN: ${{ secrets.OPENSHIFT4_SA_TOKEN }}
-          OPENSHIFT_REPOSITORY: ${{ secrets.OPENSHIFT4_REPOSITORY }}
-          TAG_NAME: ${{ env.TAG_NAME }}
-        run: |
-          make cd
-
-      - name: Rocket.Chat Notification
-        uses: RocketChat/Rocket.Chat.GitHub.Action.Notification@master
-        if: failure()
-        with:
-          type: ${{ job.status }}
-          job_name: "*Notebook Report Job Built and Deployed to ${{env.TAG_NAME}}*"
-          channel: "#registries-bot"
-          url: ${{ secrets.ROCKETCHAT_WEBHOOK }}
-          commit: true
-          token: ${{ secrets.GITHUB_TOKEN }}
+  notebook-report-cd:
+    uses: bcgov/bcregistry-sre/.github/workflows/backend-job-cd.yaml@main
+    with:
+      target: ${{ inputs.target }}
+      app_name: "notebook-report"
+      working_directory: "./jobs/notebook-report"
+    secrets:
+      WORKLOAD_IDENTIFY_POOLS_PROVIDER: ${{ secrets.WORKLOAD_IDENTIFY_POOLS_PROVIDER }}
+      GCP_SERVICE_ACCOUNT: ${{ secrets.GCP_SERVICE_ACCOUNT }}

.github/workflows/notebook-report-ci.yml

@@ -1,90 +1,21 @@
-name: Notebook Report Job CI
+name: Notebook Report CI
 
 on:
   pull_request:
-    types: [assigned, synchronize]
     paths:
       - "jobs/notebook-report/**"
-
+  workflow_dispatch:
 defaults:
   run:
     shell: bash
     working-directory: ./jobs/notebook-report
 
 jobs:
-  setup-job:
-    runs-on: ubuntu-20.04
-
-    if: github.repository == 'bcgov/namex'
-
-    steps:
-      - uses: actions/checkout@v3
-      - run: "true"
-
-  linting:
-    needs: setup-job
-    runs-on: ubuntu-20.04
-
-    strategy:
-      matrix:
-        python-version: [3.8]
-
-    steps:
-      - uses: actions/checkout@v3
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v1
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Install dependencies
-        run: |
-          make setup
-      - name: Lint with pylint
-        id: pylint
-        run: |
-          make pylint
-      - name: Lint with flake8
-        id: flake8
-        run: |
-          make flake8
-
-  testing:
-    needs: setup-job
-    env:
-      PG_USER: postgres
-      PG_PASSWORD: postgres
-      PG_DB_NAME: postgres
-      PG_HOST: localhost
-      PG_PORT: 5432      
-
-    runs-on: ubuntu-20.04
-    steps:
-      - uses: actions/checkout@v3
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v1
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Install dependencies
-        run: |
-          make setup
-      - name: Test with pytest
-        id: test
-        run: |
-      #    make test
-      #- name: Upload coverage to Codecov
-      #  uses: codecov/codecov-action@v3
-      #  with:
-      #    file: ./queue_services/entity-pay/coverage.xml
-      #    flags: entitypay
-      #    name: codecov-entity-pay
-      #    fail_ci_if_error: true
-
-  build-check:
-    needs: setup-job
-    runs-on: ubuntu-20.04
-
-    steps:
-      - uses: actions/checkout@v3
-      - name: build to check strictness
-        id: build
-        run: |
-          make build-nc
+  notebook-report-ci:
+    uses: bcgov/bcregistry-sre/.github/workflows/backend-ci.yaml@main
+    with:
+      app_name: "notebook-report"
+      working_directory: "./jobs/notebook-report"
+      codecov_flag: "notebookreport"
+      skip_isort: "true"
+      skip_black: "true"

jobs/notebook-report/.env.smample

@@ -0,0 +1,17 @@
+export NAMEX_DATABASE_USERNAME=
+export NAMEX_DATABASE_PASSWORD=
+export NAMEX_DATABASE_NAME=
+export NAMEX_DATABASE_HOST=
+export NAMEX_DATABASE_PORT=
+
+export APP_FILE=
+export DAILY_REPORT_RECIPIENTS=           #- comma separated without any space between recipient emails
+export WEEKLY_REPORT_NAMEX_RECIPIENTS=    #- comma separated without any space between recipient emails
+export WEEK_REPORT_DATE=                  #- e.g [0,1] means running on Monday and Tuesday
+export ERROR_EMAIL_RECIPIENTS=            #- comma separated without any space between recipient emails
+export ENVIRONMENT=
+
+export NOTIFY_API_URL=
+export NOTIFY_CLIENT_ID=
+export NOTIFY_CLIENT_SECRET=
+export KEYCLOAK_AUTH_TOKEN_URL=

jobs/notebook-report/Dockerfile

@@ -1,4 +1,5 @@
-FROM python:3.8.5-buster
+FROM python:3.12.2
+USER root
 
 ARG VCS_REF="missing"
 ARG BUILD_DATE="missing"
@@ -8,28 +9,74 @@ ENV BUILD_DATE=${BUILD_DATE}
 
 LABEL org.label-schema.vcs-ref=${VCS_REF} \
     org.label-schema.build-date=${BUILD_DATE}
-    
+
 USER root
 
-# Create working directory
-RUN mkdir /opt/app-root && chmod 755 /opt/app-root
-RUN mkdir /opt/app-root/data && chmod 777 /opt/app-root/data
-WORKDIR /opt/app-root
+ARG APP_ENV \
+    # Needed for fixing permissions of files created by Docker:
+    UID=1000 \
+    GID=1000
+
+ENV APP_ENV=${APP_ENV} \
+    # python:
+    PYTHONFAULTHANDLER=1 \
+    PYTHONUNBUFFERED=1 \
+    PYTHONHASHSEED=random \
+    PYTHONDONTWRITEBYTECODE=1 \
+    # pip:
+    PIP_NO_CACHE_DIR=1 \
+    PIP_DISABLE_PIP_VERSION_CHECK=1 \
+    PIP_DEFAULT_TIMEOUT=100 \
+    PIP_ROOT_USER_ACTION=ignore \
+    # poetry:
+    POETRY_VERSION=1.3.2 \
+    POETRY_NO_INTERACTION=1 \
+    POETRY_VIRTUALENVS_CREATE=false \
+    POETRY_CACHE_DIR='/var/cache/pypoetry' \
+    POETRY_HOME='/usr/local'
+
+SHELL ["/bin/bash", "-eo", "pipefail", "-c"]
+
+# Install system dependencies
+RUN apt-get update && apt-get upgrade -y \
+    && apt-get install --no-install-recommends -y \
+    bash \
+    brotli \
+    build-essential \
+    curl \
+    gettext \
+    git \
+    libpq-dev \
+    wait-for-it \
+    && curl -sSL 'https://install.python-poetry.org' | python - \
+    && poetry --version \
+    && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \
+    && apt-get clean -y && rm -rf /var/lib/apt/lists/*
 
-# Install the requirements
-COPY ./requirements.txt .
+# Setup the application directory
+WORKDIR /code
+RUN groupadd -g "${GID}" -r web \
+    && useradd -d '/code' -g web -l -r -u "${UID}" web \
+    && chown web:web -R '/code'
 
-#RUN pip install --upgrade pip
-RUN pip install pip==20.3.3
-RUN pip install --no-cache-dir -r requirements.txt
+# Copy the application code and scripts
+COPY --chown=web:web . /code
+RUN chmod 755 /code/run.sh
 
-COPY . .
+# Echo the current Poetry configuration
+RUN echo "Checking Poetry configurations..." \
+    && poetry config --list
 
-USER 1001
+# Initialize project and install dependencies
+RUN --mount=type=cache,target="$POETRY_CACHE_DIR" \
+    echo "Installing dependencies under APP_ENV=$APP_ENV..." \
+    && poetry run pip install -U pip \
+    && poetry install $(if [ -z ${APP_ENV+x} ] || [ "$APP_ENV" = 'production' ]; then echo '--only main'; fi) \
+    --no-interaction --no-ansi
 
-# Set Python path
-ENV PYTHONPATH=/opt/app-root/src
+# Switch to the non-root user
+USER web
 
 EXPOSE 8080
 
-CMD [ "python", "/opt/app-root/notebookreport.py" ]
+CMD [ "/bin/sh", "/code/run.sh" ]