Removing dev and staging Kubernetes environments | Installing Dragonf…

…ly, Elasticsearch, Apache Kafka, Debezium and Postgres in Kubernetes | Shifting from DigitalOcean to UpCloud | Some other minor changes

.gitignore

@@ -81,4 +81,9 @@ target/
 *.out
 
 # Dependency directories (remove the comment below to include it)
-vendor/
+vendor/
+
+# --- KUBERNETES ---
+
+secret.yaml
+*.secret.yaml

backend/lib.rs

@@ -202,7 +202,7 @@ pub mod utils {
     use anyhow::{anyhow, Result};
     use kafka::consumer::{Consumer, FetchOffset, GroupOffsetStorage};
     use serde::Deserialize;
-    use tracing::info;
+    use tracing::debug;
 
     pub fn createKafkaConsumer(hosts: Vec<String>, topic: String, group: String) -> Consumer {
       let consumer = Consumer::from_hosts(hosts)
@@ -216,7 +216,7 @@ pub mod utils {
           topic
         ));
 
-      info!("Created Kafka consumer for {} topic", topic);
+      debug!("Created Kafka consumer for {} topic", topic);
 
       consumer
     }

kubernetes/base/application-controller.yaml → kubernetes/manifests/application-controller.application.yaml

@@ -15,9 +15,9 @@ spec:
   source:
     repoURL: https://github.com/Archisman-Mridha/instagram-clone
     targetRevision: HEAD
-    path: kubernetes/base/application-controller
+    path: kubernetes/manifests/application-controller
 
   syncPolicy:
     automated:
       prune: true
-      selfHeal: true
+      selfHeal: true

kubernetes/base/application-controller/deployment.yaml → kubernetes/manifests/application-controller/deployment.yaml

@@ -5,6 +5,7 @@ metadata:
   namespace: default
 
 spec:
+  replicas: 1
   selector:
     matchLabels:
       app: application-controller
@@ -23,4 +24,4 @@ spec:
               cpu: 100m
             limits:
               memory: 256Mi
-              cpu: 200m
+              cpu: 200m

kubernetes/base/application-controller/kustomization.yaml → kubernetes/manifests/application-controller/kustomization.yaml

@@ -1,7 +1,6 @@
 resources:
   - instagramclone.io_applications.yaml
-
   - cluster-role.yaml
   - cluster-role-binding.yaml
   - service-account.yaml
-  - deployment.yaml
+  - deployment.yaml

kubernetes/workspaces/production/cert-manager.yaml → kubernetes/manifests/cert-manager.application.yaml

@@ -14,11 +14,11 @@ spec:
   source:
     repoURL: https://github.com/Archisman-Mridha/instagram-clone
     targetRevision: HEAD
-    path: kubernetes/workspaces/production/cert-manager
+    path: kubernetes/manifests/cert-manager
     plugin:
       name: kustomize-enable-helm
 
   syncPolicy:
     automated:
       prune: true
-      selfHeal: true
+      selfHeal: true

kubernetes/manifests/cloudnative-pg.application.yaml

@@ -0,0 +1,25 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+
+metadata:
+  name: cloudnative-pg
+  namespace: argocd
+
+spec:
+  project: default
+
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: cloudnative-pg
+
+  source:
+    repoURL: https://github.com/Archisman-Mridha/instagram-clone
+    targetRevision: HEAD
+    path: kubernetes/manifests/cloudnative-pg
+    plugin:
+      name: kustomize-enable-helm
+
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true

kubernetes/manifests/cloudnative-pg/admin-credentials.sealed-secret.yaml

@@ -0,0 +1,21 @@
+---
+apiVersion: bitnami.com/v1alpha1
+kind: SealedSecret
+metadata:
+  annotations:
+    sealedsecrets.bitnami.com/cluster-wide: "true"
+  creationTimestamp: null
+  name: admin-credentials
+  namespace: cloudnative-pg
+spec:
+  encryptedData:
+    password: AgB5Jh9QBok6cVlIGaHg0JoYKT2z/9qAzT48Y7sBoAD0NYnmga8rf+vofih0mAOEvy0q0PnZpurLCgduyuEkYgM3KVVIOBJu2bs+TB6Wgrhrm6oS+ldP790riqBmaMrnwMBuwXLVYlCHaGoR4C1CAl7M1ncFojVRA89bgRLMC2PImMzrXvBNXy4egrexLBIBN6AIjM9eTXxGSx6WlviQs8JagrqpCeswx1cnMMmIEhMzFACw33S0h04tQP0tP+6oE+1pkuM90rnBc+K6h6yo/aUBV1kdeumEbxy3Jsp1nm/9IaXHRy32h056GWyGURfJ444yZlnDa+tMoFQtYBDnq+xlBWUYEvENLNQKjdwHBjAPXWZ8Ol8RVnGv4BCu+FwjHjLCeYCpQuGzxV0A5IfLVrV37rwJhIbRhDoSJkyGqhL9i2gyZQVgZcCwNqqOPRvqioC5WENOSgNtUk44pS3p/4NHXbGe5D6/d/hy14RGNI8FsO42Abxto0ubrYF4TbuJYR39Eu4CJahyof/soX1FdgEyyY92sdmiWzormuunZN14r9NLE6hDFNIG62m8konagUtbe2/NEKdFDS1jzFJMiBwfCRdpq6Iqq1uWnkzlEweXmOpT20WsbKIt1vC8g6KmJSlKdfcT1gLUsJzKDVQrOTvi6xO7h8+hhj2SrLIhLkJK3AW3uRzXVQdpjydwJJR1UrYRgGy7LULdIGU/MGhy3vU=
+    username: AgBK6l5XgOzEk70LGaXCGwOSS/LWB34VpWf7MbqJoTQj1WulZF9Ddp26W8r9dQMzMVfjacAoGeEUrGWUa4i1KjooFr3gYS0HMyIhYmNSsZefi3+zyg6hCFk33gz80WCXpQNhy0Ogk/RpvJUWqwK3uiNlVkkHjnAyMDdtV/ujDGJGCDGfJm/aShs1YhmjDJZV504RPAjF9tvAVR5BCi53RxE47KNKD8SLI5ZNfKpNDy3O4cNxPo27B28bcJ/M6b+S/mARR8V8EhJBeN+QSKnNwj2nK5AeyPrfTohOObQNEBpJU40bpQXQccbR+TqEhAE+zTmx/R1eUHqWTDxkj7DdhaWTWEleBRnlRuiW++2QgNaqcYBTMj5/v9XOGaOY0tGxMavuO8uzeEci62+ANVWX6M2BUXUj0C/5Euv8JSSktEA3GGrJ6yHNM+xcYDIyHj9DXGqoHfbJDlxfA6hrd0Zs3p8BAS0WqHotAwynawVGblVNfjpg+OBmcE6ORJS7FMV3c7aWTBtHYYb/k5XBXUtpG9FxLjmvhsFVwLcY0xkkFXBxjRMCKJdEjDr5XOtRZcwGj1Tzj3h9//GvX789zR4wx20WWt7VEcCYJdPeiWEnGPeQpUtJqQYg9zP/wAgBZX5f9ytWk1xwi4whd4NQFcTv7ANbHCHMSsng+yKOQ/KMxvVxDpnTvj7/nBreyKG65ooxVgxIV1oxLA==
+  template:
+    metadata:
+      annotations:
+        sealedsecrets.bitnami.com/cluster-wide: "true"
+      creationTimestamp: null
+      name: admin-credentials
+      namespace: cloudnative-pg
+    type: kubernetes.io/basic-auth

kubernetes/manifests/cloudnative-pg/atlasgo-credentials.sealed-secret.yaml

@@ -0,0 +1,20 @@
+---
+apiVersion: bitnami.com/v1alpha1
+kind: SealedSecret
+metadata:
+  annotations:
+    sealedsecrets.bitnami.com/cluster-wide: "true"
+  creationTimestamp: null
+  name: atlasgo-credentials
+  namespace: cloudnative-pg
+spec:
+  encryptedData:
+    url: AgCgxs5fqw/k99uChn6XgabDowZoVS+QL0NmUcNDVupUMoVe2pIanpe1vailmbjffi+DddreBCEq4mGsgZ7OBhPGPVaIz5+k2YLMB9AeLBD4pFygdd0xPLlYBISxmdV8JTSuTu6bTrOKS9WPc/JtjrIwvo5NQDkUmUqvjXQU6TeZHO8sIVNn7b4h2kdo9b8wkOkGxCVge5k4UhB2rTJjj+sqMLio8Db0TdXXQcQ7yvcrWNc8OdxA4YxdE5eojhZw45H+hKi7EdjbofZWsE2BB/FmUel55UXnjis3m1c4w16WUzyM8YOVN99Wpke1X4C0xqtrWAuqLS6Wa2pIjUmPztbZqsnSDaW1ZAT1uoAxFSVyWSDpFaZubQcm0GqDoWIce0dp7OQ7Fdn6AHCuBj4JF5g4NXl8n0QO3HvntC7etMIZISHkFeBJLVJzWyWNBktJcbIW37Olbl+gNVGWptg1aA7rDMC35kHtA29Tw0NDuppHZcpIKmGo1iDVM51q3RDdfNJU0AokGMUu+vGc/tA7hwUl4gkFoQ84iDk18jXDjwKHSH85Qhq5xAwbeSbNJ/kijbP2DvrKBJOW/d1OvMdSYRmKyj0cuJvn6ucJ4w74eNTLaNPGiDrNjPOj2ExB4ux3BLtBRnu2BEQn2OSc8LbZviXQKgBYI+YJWCGfkovq/CqAKWoD487RGWeAStG6qXsb1V4P+1NMP55lErdc0o2O1EtPvGgA9WnzYHk+TJy86lKqQbR6n3ORnKWkQoxD1H7goU2vPndMeoG05Sbt0wg5y60fFsS9vBwgzrgpVYmgKw==
+  template:
+    metadata:
+      annotations:
+        sealedsecrets.bitnami.com/cluster-wide: "true"
+      creationTimestamp: null
+      name: atlasgo-credentials
+      namespace: cloudnative-pg
+    type: Opaque

kubernetes/manifests/cloudnative-pg/cluster.yaml

@@ -0,0 +1,73 @@
+apiVersion: postgresql.cnpg.io/v1
+
+# Represents a PostgreSQL cluster made up of a single primary and an optional number of replicas
+# that co-exist in the same Kubernetes namespace for High Availability and offloading of read-only
+# queries.
+kind: Cluster
+
+metadata:
+  name: main
+  namespace: cloudnative-pg
+
+spec:
+  instances: 1
+
+  storage:
+    pvcTemplate:
+      accessModes:
+        - ReadWriteOnce
+      resources:
+        requests:
+          storage: 1Gi
+      storageClassName: upcloud-block-storage-hdd
+      volumeMode: Filesystem
+
+  resources:
+    requests:
+      memory: 256Mi
+      cpu: 200m
+    limits:
+      memory: 256Mi
+      cpu: 200m
+
+  primaryUpdateStrategy: unsupervised
+
+  postgresql:
+    parameters:
+      # Feature to manage the retention of WAL files on the primary server. When a standby server
+      # connects to a primary server for replication, the primary server keeps track of the WAL
+      # files that the standby server has successfully replicated. Replication slots (named
+      # positions in the WAL stream) ensure that the primary server retains the necessary WAL files
+      # until all standby servers have consumed them. This prevents the primary server from
+      # removing WAL files that are still needed by the standby servers for replication, thus
+      # avoiding potential data loss scenarios.
+      max_replication_slots: "4"
+
+      # WAL senders are PostgreSQL server processes responsible for streaming WAL data from the
+      # primary server to standby servers for replication. When a standby server establishes a
+      # streaming replication connection to the primary server, the primary server creates a WAL
+      # sender process dedicated to streaming WAL data to that specific standby server.
+      #
+      # Determines how many maximum standby servers can simultaneously replicate from the primary
+      # server.
+      max_wal_senders: "4"
+
+      # How much memory is dedicated to the PostgreSQL server for caching data.
+      shared_buffers: 64MB
+
+  bootstrap:
+    initdb:
+      database: instagram_clone
+
+      owner: admin
+      secret:
+        name: admin-credentials
+
+      postInitApplicationSQLRefs:
+        secretRefs:
+          - name: main-cluster-init-sql
+            key: create-debezium-user.sql
+
+  # Enabling integration with Prometheus and Grafana.
+  monitoring:
+    enablePodMonitor: true

kubernetes/manifests/cloudnative-pg/kustomization.yaml

@@ -0,0 +1,21 @@
+resources:
+  - namespace.yaml
+  - main-cluster-init-sql.sealed-secret.yaml
+  - cluster.yaml
+  - admin-credentials.sealed-secret.yaml
+  - grafana-dashboard.configmap.yaml
+  - atlasgo-credentials.sealed-secret.yaml
+  - schema.yaml
+
+helmCharts:
+  - repo: https://cloudnative-pg.io/charts
+    name: cloudnative-pg
+    releaseName: cloudnative-pg
+    namespace: cloudnative-pg
+    version: 0.20.0
+
+  - repo: oci://ghcr.io/ariga/charts
+    name: atlas-operator
+    releaseName: atlas
+    namespace: cloudnative-pg
+    version: 0.4.0

kubernetes/manifests/cloudnative-pg/main-cluster-init-sql.sealed-secret.yaml

@@ -0,0 +1,20 @@
+---
+apiVersion: bitnami.com/v1alpha1
+kind: SealedSecret
+metadata:
+  annotations:
+    sealedsecrets.bitnami.com/cluster-wide: "true"
+  creationTimestamp: null
+  name: main-cluster-init-sql
+  namespace: cloudnative-pg
+spec:
+  encryptedData:
+    create-debezium-user.sql: AgCimJt9zOO3i8eRCYET8UYIGCQdS28PXv0y+8brA4oEi4wu+WTvkRbi4AmqPqMPXMWyvxEGtMig0GJwu3VTRj+Lc9nCyt9Kb5L5KdGy60eurAx8e7SCgvmiMeG+RX5D5uaQpeRCBof5LvDy1Eq/m8ztJjT6BJq2BCjNTaWP3A/1dEq4soUCMqNbaN0lt4iXxvrCTNeGgeHOFt7TQwnhwIFkCaYBBg6iR6SE28VfHE7v1S+oB7+bSIxQkRUmVklaq9M1q8XQ2L///50CJhFRVhqc/+u4chl73PHeSQ3cykLEZrNyH5Wft3NsQYdR3zmuBSvRqsLQV/V5u/YnczN2bYf599sbn1K/pX6MVhoMc4gju2n0qtVMg+qg0G0sE08eOwJ302v/J+312iUjKib1ZVdtwJKNA5EcBPCVxdUbrXhGo3kRd/xocGw9S5M+CaiOpPCRLrpRF7wWcC3mj4SM6X4dPGZXpQNFE6bDI/Oxg2vhYVSAJ1cM2VN5mJYZDLelg8kyyqjeCAVyg3IshfE1Zc3tN/oIvOSujT/7m4tv79keaK8ojube3CDGgYFA29NVWkL3aWnMyTYkrZ6yQMKpV80n6IcJU4vFee7Wbkiu6BkZTsVV6FblfrW+EBNugswMlWkEYk8G9l8UUvoLNuFZ9kPmK6APQa+Dokt/PKGHGShH9IXQr73vtzBxfGEe5q5fzpDtpFNYXy+i3MqkbzR5tI2HGbk4PZMBM+ogYpr1tbPIq6uqQheGKLJLj9P6ClmGV3iJWrbl4XxMwKb1RvW2Z8MZCLDGDIYcIyYqzW4fTRDlam85tUDaptyfeDm2
+  template:
+    metadata:
+      annotations:
+        sealedsecrets.bitnami.com/cluster-wide: "true"
+      creationTimestamp: null
+      name: main-cluster-init-sql
+      namespace: cloudnative-pg
+    type: Opaque

kubernetes/manifests/cloudnative-pg/namespace.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: cloudnative-pg
+  labels:
+    pod-monitor-scanning: enable

kubernetes/manifests/cloudnative-pg/schema.yaml

@@ -0,0 +1,57 @@
+apiVersion: db.atlasgo.io/v1alpha1
+kind: AtlasSchema
+metadata:
+  name: instagram-clone
+  namespace: cloudnative-pg
+
+spec:
+
+  urlFrom:
+    secretKeyRef:
+      key: url
+      name: atlasgo-credentials
+
+  schema:
+    sql: |
+
+      -- Users Microservice
+      CREATE TABLE users (
+        id SERIAL PRIMARY KEY,
+
+        name VARCHAR(25) NOT NULL,
+        email VARCHAR(320) NOT NULL UNIQUE,
+        username VARCHAR(25) NOT NULL UNIQUE,
+
+        password VARCHAR(100) NOT NULL
+      );
+      CREATE INDEX email_idx_users ON users (email);
+      CREATE INDEX username_idx_users ON users (username);
+
+      -- Profiles Microservice
+      CREATE TABLE profiles (
+        id INT PRIMARY KEY,
+
+        name VARCHAR(25) NOT NULL,
+        username VARCHAR(25) NOT NULL UNIQUE,
+
+        profile_picture_uri VARCHAR(250)
+      );
+      CREATE INDEX username_idx_profiles ON profiles (username);
+
+      -- Followships Microservice
+      CREATE TABLE followships (
+        follower_id INT,
+        followee_id INT
+      );
+      CREATE INDEX follower_id_idx_followships ON followships (follower_id, followee_id);
+      CREATE INDEX followee_id_idx_followships ON followships (followee_id, follower_id);
+
+      -- Posts Microservice
+      CREATE TABLE posts (
+        id SERIAL PRIMARY KEY,
+        owner_id INT NOT NULL,
+
+        description VARCHAR(255) NOT NULL,
+        created_at TIMESTAMP NOT NULL DEFAULT now( )
+      );
+      CREATE INDEX owner_id_idx_posts ON posts (owner_id, created_at DESC);

kubernetes/workspaces/dev/istio.yaml → kubernetes/manifests/debezium.application.yaml

@@ -2,22 +2,22 @@ apiVersion: argoproj.io/v1alpha1
 kind: Application
 
 metadata:
-  name: istio
-  namespace: istio-system
+  name: debezium
+  namespace: argocd
 
 spec:
   project: default
 
   destination:
     server: https://kubernetes.default.svc
-    namespace: istio-system
+    namespace: debezium
 
   source:
     repoURL: https://github.com/Archisman-Mridha/instagram-clone
     targetRevision: HEAD
-    path: kubernetes/workspaces/dev/istio
+    path: kubernetes/manifests/debezium
 
   syncPolicy:
     automated:
       prune: true
-      selfHeal: true
+      selfHeal: true

kubernetes/manifests/debezium/kustomization.yaml

@@ -0,0 +1,13 @@
+resources:
+  # CRDs
+  - https://raw.githubusercontent.com/debezium/debezium-operator/2.4/k8/debeziumservers.debezium.io-v1.yml
+
+  # Operator
+  - https://raw.githubusercontent.com/debezium/debezium-operator/2.4/k8/kubernetes.yml
+
+  - namespace.yaml
+
+  - postgres-credentials.sealed-secret.yaml
+
+  - users-table.debezium-server.yaml
+  - posts-table.debezium-server.yaml

kubernetes/workspaces/dev/kube-prometheus-stack/namespace.yaml → kubernetes/manifests/debezium/namespace.yaml

@@ -1,4 +1,4 @@
 apiVersion: v1
 kind: Namespace
 metadata:
-  name: prometheus
+  name: debezium