Merge branch 'refactor/noti-router'

noi-techpark · Feb 6, 2025 · 772beab · 772beab
2 parents a3bb2a1 + 1f1b14b
commit 772beab
Show file tree

Hide file tree

Showing 68 changed files with 1,465 additions and 3,590 deletions.
diff --git a/.github/workflows/odh-v2-core-bridge.yaml b/.github/workflows/odh-v2-core-bridge.yaml
@@ -0,0 +1,58 @@
+name: CI/CD odh-v2-core-writer
+
+on: 
+  push:
+    paths:
+      - "infrastructure/raw-data-bridge/**"      
+      - "infrastructure/helm/raw-data-bridge/**"
+      - ".github/workflows/odh-v2-core-writer.yaml"
+  workflow_dispatch:
+
+env:
+  WORKING_DIRECTORY: infrastructure/raw-data-bridge
+  KUBERNETES_NAMESPACE: core
+  KUBERNETES_VALUE_PATH: infrastructure/helm/raw-data-bridge/values.yaml
+  KUBERNETES_CHART_PATH: infrastructure/helm/raw-data-bridge/raw-data-bridge
+  PROJECT_NAME: raw-data-bridge
+
+jobs:
+  build:
+    if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/prod'
+    runs-on: ubuntu-22.04
+    concurrency: odh-v2-core-writer-build
+    steps:
+    - name: Checkout source code
+      uses: actions/checkout@v4
+
+    - name: Log in to GitHub Docker Registry
+      uses: docker/login-action@v2
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Manually build and push image
+      run: cd ${{ env.WORKING_DIRECTORY }} && ./build.sh
+
+  deploy:
+    if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/prod'
+    needs: 
+      - build
+    runs-on: ubuntu-22.04
+    concurrency: odh-v2-core-writer-build-deploy
+    environment: ${{ github.ref == 'refs/heads/main' && 'test' || github.ref == 'refs/heads/prod' && 'prod' || 'unknown' }}
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v4
+
+      - name: Deploy on cluster
+        uses: noi-techpark/github-actions/helm-deploy@v2
+        with:
+          k8s-name: ${{ env.PROJECT_NAME }}
+          k8s-namespace: ${{ env.KUBERNETES_NAMESPACE }}
+          chart-path: ${{ env.KUBERNETES_CHART_PATH }}
+          values-file: ${{ env.KUBERNETES_VALUE_PATH }}
+          aws-access-key-id: ${{ secrets[vars.AWS_KEY_ID] }}
+          aws-secret-access-key: ${{ secrets[vars.AWS_KEY_SECRET] }}
+          aws-eks-cluster-name: aws-main-eu-01
+          aws-region: eu-west-1
diff --git a/.github/workflows/odh-v2-core-router.yaml b/.github/workflows/odh-v2-core-router.yaml
@@ -0,0 +1,58 @@
+name: CI/CD odh-v2-core-router
+
+on: 
+  push:
+    paths:
+      - "infrastructure/router/**"      
+      - "infrastructure/helm/router/**"
+      - ".github/workflows/odh-v2-core-router.yaml"
+  workflow_dispatch:
+
+env:
+  WORKING_DIRECTORY: infrastructure/router
+  KUBERNETES_NAMESPACE: core
+  KUBERNETES_VALUE_PATH: infrastructure/helm/router/values.yaml
+  KUBERNETES_CHART_PATH: infrastructure/helm/router/router
+  PROJECT_NAME: router
+
+jobs:
+  build:
+    if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/prod'
+    runs-on: ubuntu-22.04
+    concurrency: odh-v2-core-router-build
+    steps:
+    - name: Checkout source code
+      uses: actions/checkout@v4
+
+    - name: Log in to GitHub Docker Registry
+      uses: docker/login-action@v2
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Manually build and push image
+      run: cd ${{ env.WORKING_DIRECTORY }} && ./build.sh
+
+  deploy:
+    if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/prod'
+    needs: 
+      - build
+    runs-on: ubuntu-22.04
+    concurrency: odh-v2-core-router-build-deploy
+    environment: ${{ github.ref == 'refs/heads/main' && 'test' || github.ref == 'refs/heads/prod' && 'prod' || 'unknown' }}
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v4
+
+      - name: Deploy on cluster
+        uses: noi-techpark/github-actions/helm-deploy@v2
+        with:
+          k8s-name: ${{ env.PROJECT_NAME }}
+          k8s-namespace: ${{ env.KUBERNETES_NAMESPACE }}
+          chart-path: ${{ env.KUBERNETES_CHART_PATH }}
+          values-file: ${{ env.KUBERNETES_VALUE_PATH }}
+          aws-access-key-id: ${{ secrets[vars.AWS_KEY_ID] }}
+          aws-secret-access-key: ${{ secrets[vars.AWS_KEY_SECRET] }}
+          aws-eks-cluster-name: aws-main-eu-01
+          aws-region: eu-west-1
diff --git a/.github/workflows/odh-v2-core-writer.yaml b/.github/workflows/odh-v2-core-writer.yaml
@@ -0,0 +1,58 @@
+name: CI/CD odh-v2-core-writer
+
+on: 
+  push:
+    paths:
+      - "infrastructure/raw-writer/**"      
+      - "infrastructure/helm/raw-writer/**"
+      - ".github/workflows/odh-v2-core-writer.yaml"
+  workflow_dispatch:
+
+env:
+  WORKING_DIRECTORY: infrastructure/raw-writer
+  KUBERNETES_NAMESPACE: core
+  KUBERNETES_VALUE_PATH: infrastructure/helm/raw-writer/values.yaml
+  KUBERNETES_CHART_PATH: infrastructure/helm/raw-writer/raw-writer
+  PROJECT_NAME: raw-writer
+
+jobs:
+  build:
+    if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/prod'
+    runs-on: ubuntu-22.04
+    concurrency: odh-v2-core-writer-build
+    steps:
+    - name: Checkout source code
+      uses: actions/checkout@v4
+
+    - name: Log in to GitHub Docker Registry
+      uses: docker/login-action@v2
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Manually build and push image
+      run: cd ${{ env.WORKING_DIRECTORY }} && ./build.sh
+
+  deploy:
+    if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/prod'
+    needs: 
+      - build
+    runs-on: ubuntu-22.04
+    concurrency: odh-v2-core-writer-build-deploy
+    environment: ${{ github.ref == 'refs/heads/main' && 'test' || github.ref == 'refs/heads/prod' && 'prod' || 'unknown' }}
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v4
+
+      - name: Deploy on cluster
+        uses: noi-techpark/github-actions/helm-deploy@v2
+        with:
+          k8s-name: ${{ env.PROJECT_NAME }}
+          k8s-namespace: ${{ env.KUBERNETES_NAMESPACE }}
+          chart-path: ${{ env.KUBERNETES_CHART_PATH }}
+          values-file: ${{ env.KUBERNETES_VALUE_PATH }}
+          aws-access-key-id: ${{ secrets[vars.AWS_KEY_ID] }}
+          aws-secret-access-key: ${{ secrets[vars.AWS_KEY_SECRET] }}
+          aws-eks-cluster-name: aws-main-eu-01
+          aws-region: eu-west-1
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -5,12 +5,28 @@
     "version": "0.2.0",
     "configurations": [
         {
-            "name": "Launch file",
+            "name": "Launch Writer",
             "type": "go",
             "request": "launch",
             "mode": "debug",
-            "program": "${file}",
-            "envFile": "${fileDirname}/.env"
+            "program": "${workspaceFolder}/infrastructure/raw-writer",
+            "envFile": "${workspaceFolder}/infrastructure/raw-writer/.env"
+        },
+        {
+            "name": "Launch Router",
+            "type": "go",
+            "request": "launch",
+            "mode": "debug",
+            "program": "${workspaceFolder}/infrastructure/router",
+            "envFile": "${workspaceFolder}/infrastructure/router/.env"
+        },
+        {
+            "name": "Launch Lookup",
+            "type": "go",
+            "request": "launch",
+            "mode": "debug",
+            "program": "${workspaceFolder}/infrastructure/raw-data-bridge",
+            "envFile": "${workspaceFolder}/infrastructure/raw-data-bridge/.env"
         }
     ]
 }
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -52,10 +52,12 @@ services:
   # Writer service listens on ingress queue and writes raw data to mongodb
   writer:
     image: ghcr.io/noi-techpark/infrastructure-v2/raw-writer:latest
+    pull_policy: always
     environment:
       APP_MQ_URI: amqp://guest:guest@rabbitmq:5672
       APP_MQ_EXCHANGE: ingress
       APP_MQ_QUEUE: ingress
+      APP_MQ_READY_EXCHANGE: ready
       APP_MONGO_URI: mongodb://mongodb/?replicaSet=rs0&directConnection=true
       APP_LOGLEVEL: INFO
     depends_on:
@@ -64,42 +66,21 @@ services:
       rabbitmq:
         condition: service_healthy
 
-  # Notifier service listens on mongodb oplog and generates an event when data is inserted
-  notifier:
-    image: ghcr.io/noi-techpark/infrastructure-v2/notifier:latest
-    depends_on:
-      mongodb:
-        condition: service_healthy
-      rabbitmq:
-        condition: service_healthy
-      mongodb-init:
-        condition: service_completed_successfully
-    environment:
-      MONGODB_CONNECTION_STRING: mongodb://mongodb:27017?directConnection=true
-      MONGODB_CONNECTION_REPLICA_SET: rs0
-      RABBITMQ_CLUSTER_URL: amqp://guest:guest@rabbitmq:5672/
-
-  # Router service listens on ready exchange (where notifier writes) and distributes messages according to routing key
+  # Router service listens on ready exchange (where writer writes) and distributes messages according to routing key
   router:
-    build:
-      context: infrastructure/router
-      dockerfile: Dockerfile
-      target: dev
+    image: ghcr.io/noi-techpark/infrastructure-v2/router:latest
+    pull_policy: always
     environment:
-      RABBITMQ_CLUSTER_URL: rabbitmq:5672
-      RABBITMQ_USER: guest
-      RABBITMQ_PASSWORD: guest
-      RABBITMQ_CLIENTNAME: writer
-    volumes:
-      - ./infrastructure/router:/code
-      - mvn-cache:/code/.mvn
+      APP_MQ_URI: amqp://guest:guest@rabbitmq:5672
+      APP_MQ_READY_EXCHANGE: ready
+      APP_MQ_READY_QUEUE: ready
+      APP_MQ_ROUTED_EXCHANGE: routed
+      APP_MQ_ROUTED_QUEUE: routed
     depends_on:
-      - notifier
-    working_dir: /code
-    command: ["mvn -B quarkus:dev"]
-
-volumes:
-  mvn-cache:
+      mongodb-init:
+        condition: service_completed_successfully
+      rabbitmq:
+        condition: service_healthy
 
 networks:
   default: