Adding Kubernetes Application resource manifests for microservices …

…| Disabling `dorny paths filter` in Github Actions | Update Readme.md
Archisman-Mridha · Jan 28, 2024 · c7590de · c7590de
1 parent e9d1acc
commit c7590de
Show file tree

Hide file tree

Showing 17 changed files with 396 additions and 63 deletions.
diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml
@@ -6,9 +6,6 @@ name: Main
 on:
   push:
     branches: [main]
-  pull_request:
-    branches: [dev, main]
-    types: [opened, edited]
 
   ## Adding this allows us to trigger this workflow manually (Just for debugging purposes).
   workflow_dispatch: {}
@@ -131,8 +128,8 @@ jobs:
                 - go.work.sum
 
           - name: application-controller
-            path: backend/gateway
-            manifest: kubernetes/base/microservices/application-controller.application.yaml
+            path: kubernetes/operators/application
+            manifest: kubernetes/base/application-controller/deployment.yaml
             path_filters: |
               changed:
                 - kubernetes/operators/application/**
@@ -144,31 +141,31 @@ jobs:
 
       ## If sourcecode of the microservice has changed, only then we will rebuild, push, sign
       ## and scan the container image.
-      - name: Detect sourcecode change
-        uses: dorny/paths-filter@v2
-        id: path-filter
-        with:
-          base: ${{ github.ref }}
-          filters: ${{ matrix.microservice.path_filters }}
+      ## - name: Detect sourcecode change
+      ##   uses: dorny/paths-filter@v2
+      ##   id: path-filter
+      ##   with:
+      ##     base: ${{ github.ref }}
+      ##     filters: ${{ matrix.microservice.path_filters }}
 
       - name: Set up QEMU
-        if: steps.path-filter.outputs.changed == 'true'
+        ## if: steps.path-filter.outputs.changed == 'true'
         uses: docker/setup-qemu-action@v2
 
       - name: Set up Docker Buildx
-        if: steps.path-filter.outputs.changed == 'true'
+        ## if: steps.path-filter.outputs.changed == 'true'
         uses: docker/setup-buildx-action@v2
 
       - name: Login to GitHub Container Registry
-        if: steps.path-filter.outputs.changed == 'true'
+        ## if: steps.path-filter.outputs.changed == 'true'
         uses: docker/login-action@v2.1.0
         with:
           registry: ghcr.io
           username: archisman-mridha
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Restore cached Cargo dependencies (if exists)
-        if: steps.path-filter.outputs.changed == 'true'
+        ## if: steps.path-filter.outputs.changed == 'true'
         uses: actions/cache/restore@v3
         with:
           path: |
@@ -177,7 +174,7 @@ jobs:
           key: ${{ runner.os }}-cargo-${{ matrix.microservice.name }}-${{ hashFiles('**/Cargo.lock') }}
 
       - name: Build and push AMD64 container image
-        if: steps.path-filter.outputs.changed == 'true'
+        ## if: steps.path-filter.outputs.changed == 'true'
         uses: docker/build-push-action@v4
         with:
           context: .
@@ -194,7 +191,7 @@ jobs:
           cache-to: type=gha,mode=max
 
       - name: Cache Cargo dependencies
-        if: steps.path-filter.outputs.changed == 'true'
+        ## if: steps.path-filter.outputs.changed == 'true'
         uses: actions/cache@v3
         with:
           path: |
@@ -203,20 +200,20 @@ jobs:
           key: ${{ runner.os }}-cargo-${{ matrix.microservice.name }}-${{ hashFiles('**/Cargo.lock') }}
 
       - name: Remove cached folders from local machine
-        if: steps.path-filter.outputs.changed == 'true'
+        ## if: steps.path-filter.outputs.changed == 'true'
         run: |
           rm -rf /usr/local/cargo/registry/ target/
 
       ## Cosign is a command line utility that can sign and verify software artifact, such as
       ## container images and blobs.
       - name: Install Cosign
-        if: steps.path-filter.outputs.changed == 'true'
+        ## if: steps.path-filter.outputs.changed == 'true'
         uses: sigstore/cosign-installer@v3.1.1
         with:
           cosign-release: v2.2.1
 
       - name: Sign the published container image
-        if: steps.path-filter.outputs.changed == 'true'
+        ## if: steps.path-filter.outputs.changed == 'true'
         env:
           COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }}
         run: |
@@ -228,11 +225,11 @@ jobs:
             ghcr.io/archisman-mridha/instagram-clone-${{ matrix.microservice.name }}:${{ github.sha }} -y
 
       - name: Create outputs directory
-        if: steps.path-filter.outputs.changed == 'true'
+        ## if: steps.path-filter.outputs.changed == 'true'
         run: mkdir -p ./outputs/trivy
 
       - name: Scan container image for vulnerabilities
-        if: steps.path-filter.outputs.changed == 'true'
+        ## if: steps.path-filter.outputs.changed == 'true'
         uses: aquasecurity/trivy-action@master
         with:
           image-ref: ghcr.io/archisman-mridha/instagram-clone-${{ matrix.microservice.name }}:${{ github.sha }}
@@ -243,23 +240,23 @@ jobs:
           output: ./outputs/trivy/${{ matrix.microservice.name }}.container-image-scan-result.sarif
 
       - name: Upload the scan result as Github artifact
-        if: steps.path-filter.outputs.changed == 'true'
+        ## if: steps.path-filter.outputs.changed == 'true'
         uses: actions/upload-artifact@v3
         with:
           name: trivy.${{ matrix.microservice.name }}-microservice.container-image-scan-result.sarif
           path: ./outputs/trivy/${{ matrix.microservice.name }}.container-image-scan-result.sarif
 
-      ## - name: Update container image tag in Kubernetes manifests
-      ##   if: steps.path-filter.outputs.changed == 'true'
-      ##   run: |
-      ##     git config --global user.name "Archisman-Mridha"
-      ##     git config --global user.email "archismanmridha12345@gmail.com"
-      ##     git config --global pull.rebase false
+      - name: Update container image tag in Kubernetes manifests
+        ## if: steps.path-filter.outputs.changed == 'true'
+        run: |
+          git config --global user.name "Archisman-Mridha"
+          git config --global user.email "archismanmridha12345@gmail.com"
+          git config --global pull.rebase false
 
-      ##     git pull origin main
+          git pull origin main
 
-      ##     sed -i 's/instagram-clone-\(.*\):[[:alnum:]]\+/instagram-clone-\1:${{ github.sha }}/g' ${{ matrix.microservice.manifest }}
+          sed -i 's/instagram-clone-\(.*\):[[:alnum:]]\+/instagram-clone-\1:${{ github.sha }}/g' ${{ matrix.microservice.manifest }}
 
-      ##     git add .
-      ##     git commit -m "🤖 Update container image tag for ${{ matrix.microservice.name }} to ${{ github.sha }}"
-      ##     git push --set-upstream origin main
+          git add .
+          git commit -m "🤖 Update container image tag for ${{ matrix.microservice.name }} to ${{ github.sha }}"
+          git push --set-upstream origin main
diff --git a/README.md b/README.md
@@ -1,5 +1,7 @@
 # Instagram Clone
 
+![Main Workflow](https://github.com/Archisman-Mridha/instagram-clone/actions/workflows/main.yaml/badge.svg)
+
 Welcome to the `Instagram Clone` project, which aims to replicate the core functionalities of the popular social media platform, **Instagram**, while demonstrating :
 
 > How to build and deploy fault tolerant distributed systems.
@@ -16,9 +18,9 @@ If you encounter any bugs, have feature requests, or just want to discuss about
 ## References
 
 - [Microservices Patterns](https://www.oreilly.com/library/view/microservices-patterns/9781617294549/) book by Chris Richardson
-- [Jordan has no life](https://www.youtube.com/@jordanhasnolife5163)'s [Youtube video](https://www.youtube.com/watch?v=S2y9_XYOZsg&pp=ygUqam9yZGFuIGhhcyBubyBsaWZlIGluc3RhZ3JhbSBzeXN0ZW0gZGVzaWdu) on how to design a social media platform like Instagram
+- [Jordan has no life](https://www.youtube.com/@jordanhasnolife5163)'s Youtube video on [how to design a social media platform like Instagram](https://www.youtube.com/watch?v=S2y9_XYOZsg&pp=ygUqam9yZGFuIGhhcyBubyBsaWZlIGluc3RhZ3JhbSBzeXN0ZW0gZGVzaWdu)
 - The [Rust Book](https://doc.rust-lang.org/book/)
 - [Crust of Rust](https://youtube.com/playlist?list=PLqbS7AVVErFiWDOAVrPt7aYmnuuOLYvOa&si=-Y5Byig03EhhX0pi) Youtube playlist by [Jon Gjengset](https://www.youtube.com/@jonhoo)
 - Youtube channels - [Mario Carrion](https://www.youtube.com/@MarioCarrion/playlists) | [Viktor Farcic](https://www.youtube.com/@DevOpsToolkit)
-- [Vivek Singh](https://www.youtube.com/@viveksinghggits)'s [Youtube playlist](https://www.youtube.com/playlist?list=PLh4KH3LtJvRTtFWz1WGlyDa7cKjj2Sns0) on how to write a custom Kubernetes controller
-- [Heiko's Blogs](https://heikoseeberger.de/tags/opentelemetry/) on how to instrument Rust codebase for Distrbuted Tracing
+- [Vivek Singh](https://www.youtube.com/@viveksinghggits)'s Youtube playlist on [how to write a custom Kubernetes controller](https://www.youtube.com/playlist?list=PLh4KH3LtJvRTtFWz1WGlyDa7cKjj2Sns0)
+- [Heiko's Blogs](https://heikoseeberger.de) on [how to instrument Rust codebase for Distrbuted Tracing](https://heikoseeberger.de/tags/opentelemetry/)
diff --git a/compose.yaml b/compose.yaml
@@ -1,7 +1,6 @@
-version: '3'
+version: "3"
 
 services:
-
   postgres:
     container_name: postgres
     image: postgres:alpine
@@ -13,9 +12,28 @@ services:
     ports:
       - 5432:5432
     ## 'wal_level = logical' is the highest level of WAL logging.
-    command: ["postgres", "-c", "wal_level=logical", "-c", "max_replication_slots=4", "-c", "max_wal_senders=4"]
+    command:
+      [
+        "postgres",
+        "-c",
+        "wal_level=logical",
+        "-c",
+        "max_replication_slots=4",
+        "-c",
+        "max_wal_senders=4",
+      ]
     healthcheck:
-      test: ["CMD", "psql", "-U", "default", "-d", "instagram_clone", "-c", "SELECT 1"]
+      test:
+        [
+          "CMD",
+          "psql",
+          "-U",
+          "default",
+          "-d",
+          "instagram_clone",
+          "-c",
+          "SELECT 1",
+        ]
       interval: 10s
       timeout: 5s
       retries: 5
@@ -25,7 +43,14 @@ services:
     image: migrate/migrate:latest
     volumes:
       - ./backend/sql/schema.sql:/migrations/000001_init.up.sql:ro
-    command:  ["-path", "/migrations", "-database",  "postgres://default:pass@postgres:5432/instagram_clone?sslmode=disable", "up"]
+    command:
+      [
+        "-path",
+        "/migrations",
+        "-database",
+        "postgres://default:pass@postgres:5432/instagram_clone?sslmode=disable",
+        "up",
+      ]
     depends_on:
       postgres:
         condition: service_healthy
@@ -65,15 +90,21 @@ services:
       - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
       - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT
     healthcheck:
-      test: ["CMD", "kafka-topics.sh", "--bootstrap-server=localhost:9092", "--list"]
+      test:
+        [
+          "CMD",
+          "kafka-topics.sh",
+          "--bootstrap-server=localhost:9092",
+          "--list",
+        ]
       start_period: 15s
       interval: 10s
 
   kafka-migrate:
     container_name: kafka-migrate
     image: bitnami/kafka
     working_dir: /opt/bitnami/kafka/bin
-    entrypoint: [ "/bin/sh", "-c" ]
+    entrypoint: ["/bin/sh", "-c"]
     depends_on:
       kafka:
         condition: service_healthy
@@ -133,7 +164,16 @@ services:
       VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
       ENABLE_DEBEZIUM_SCRIPTING: "true"
     healthcheck:
-      test: ["CMD", "curl", "--silent", "--fail", "-X", "GET", "http://localhost:8083/connectors"]
+      test:
+        [
+          "CMD",
+          "curl",
+          "--silent",
+          "--fail",
+          "-X",
+          "GET",
+          "http://localhost:8083/connectors",
+        ]
       start_period: 10s
       interval: 10s
       timeout: 5s
@@ -231,28 +271,28 @@ services:
     depends_on:
       - elasticsearch
 
-  ## prometheus:
-  ##   container_name: prometheus
-  ##   image: prom/prometheus
-  ##   ports:
-  ##     - 9090:9090
-  ##   volumes:
-  ##     - ./prometheus.dev-config.yaml:/etc/prometheus/prometheus.yml:ro
+  prometheus:
+    container_name: prometheus
+    image: prom/prometheus
+    ports:
+      - 9090:9090
+    volumes:
+      - ./prometheus.dev-config.yaml:/etc/prometheus/prometheus.yml:ro
 
-  ## autometrics:
-  ##   container_name: autometrics
-  ##   image: autometrics/am-proxy
-  ##   environment:
-  ##     - PROMETHEUS_URL=http://prometheus:9090
-  ##   ports:
-  ##     - 6789:6789
+  autometrics:
+    container_name: autometrics
+    image: autometrics/am-proxy
+    environment:
+      - PROMETHEUS_URL=http://prometheus:9090
+    ports:
+      - 6789:6789
 
   jaeger:
     container_name: jaeger
     image: jaegertracing/all-in-one:latest
     hostname: jaeger
     ports:
-      - 4317:4317   ## gRPC endpoint
+      - 4317:4317 ## gRPC endpoint
       - 16686:16686 ## UI dashboard endpoint
     environment:
       - COLLECTOR_OTLP_ENABLED=true
@@ -461,4 +501,4 @@ services:
       driver: fluentd
       options:
         fluentd-address: localhost:24224
-        tag: gateway
+        tag: gateway
diff --git a/kubernetes/base/application-controller/deployment.yaml b/kubernetes/base/application-controller/deployment.yaml
@@ -16,7 +16,7 @@ spec:
       serviceAccountName: application-controller
       containers:
         - name: application-controller
-          image: ghcr.io/archisman-mridha/instagram-clone-application-controller:78cd2ead9ce9828ab6ec1265192df427eca05e90
+          image: ghcr.io/archisman-mridha/instagram-clone-application-controller:9574f877a5a231dbab94fa621fac97734ad5227e
           resources:
             requests:
               memory: 128Mi

diff --git a/kubernetes/base/microservices/feeds-microservice.application.yaml b/kubernetes/base/microservices/feeds-microservice.application.yaml
@@ -0,0 +1,16 @@
+apiVersion: instagramclone.io/v1alpha1
+kind: Application
+metadata:
+  name: feeds-microservice
+  namespace: microservices
+
+spec:
+  image: ghcr.io/archisman-mridha/instagram-clone-feeds-microservice:9574f877a5a231dbab94fa621fac97734ad5227e
+  replicas:
+    min: 2
+    max: 3
+  resources:
+    cpu: 100m
+    memory: 100Mi
+  secretName: feeds-microservice
+  port: 4000
diff --git a/kubernetes/base/microservices/followships-microservice.application.yaml b/kubernetes/base/microservices/followships-microservice.application.yaml
@@ -0,0 +1,16 @@
+apiVersion: instagramclone.io/v1alpha1
+kind: Application
+metadata:
+  name: followships-microservice
+  namespace: microservices
+
+spec:
+  image: ghcr.io/archisman-mridha/instagram-clone-followships-microservice:9574f877a5a231dbab94fa621fac97734ad5227e
+  replicas:
+    min: 2
+    max: 3
+  resources:
+    cpu: 100m
+    memory: 100Mi
+  secretName: followships-microservice
+  port: 4000
diff --git a/kubernetes/base/microservices/gateway.application.yaml b/kubernetes/base/microservices/gateway.application.yaml
@@ -0,0 +1,16 @@
+apiVersion: instagramclone.io/v1alpha1
+kind: Application
+metadata:
+  name: gateway
+  namespace: microservices
+
+spec:
+  image: ghcr.io/archisman-mridha/instagram-clone-gateway:9574f877a5a231dbab94fa621fac97734ad5227e
+  replicas:
+    min: 2
+    max: 3
+  resources:
+    cpu: 100m
+    memory: 100Mi
+  secretName: gateway
+  port: 4000