Add Tasking manager DB (#327)

* Add TM database

* Add values

* Fix tm postgres uri

* Add a section in cronjob to backup TM - DB

* Add values to create ClusterIssuer

* Update if statement

osm-seed/templates/db/db-backup-job.yaml

@@ -56,6 +56,38 @@ spec:
                 value: {{ $job.env.RESTORE_URL_FILE}}
             {{- end }}
             # Env vars for osmcha database
+            {{ if eq $job.name "tm-db" }}
+              # DB credentials
+              - name: POSTGRES_HOST
+                value: {{ $.Release.Name }}-tm-db
+              - name: POSTGRES_DB
+                value: {{ $.Values.tmDb.env.POSTGRES_DB }}
+              - name: POSTGRES_PASSWORD
+                value: {{ quote $.Values.tmDb.env.POSTGRES_PASSWORD }}
+              - name: POSTGRES_USER
+                value: {{ $.Values.tmDb.env.POSTGRES_USER }}
+              # DB action: backup/restore
+              - name: DB_ACTION
+                value: {{ quote $job.env.DB_ACTION }}
+              # Backup naming
+              - name: SET_DATE_AT_NAME
+                value: {{quote $job.env.SET_DATE_AT_NAME }}
+              - name: BACKUP_CLOUD_FOLDER
+                value: {{ $job.env.BACKUP_CLOUD_FOLDER }}
+              - name: BACKUP_CLOUD_FILE
+                value: {{ $job.env.BACKUP_CLOUD_FILE }}
+              - name: AWS_S3_BUCKET
+                value: {{ $job.env.AWS_S3_BUCKET }}
+              # Clean up options
+              - name: CLEANUP_BACKUPS
+                value: {{ quote $job.env.CLEANUP_BACKUPS }}
+              - name: RETENTION_DAYS
+                value: {{ quote $job.env.RETENTION_DAYS }}
+              # In case DB_ACTION=restore, adn the backup url
+              - name: RESTORE_URL_FILE
+                value: {{ $job.env.RESTORE_URL_FILE}}
+            {{- end }}
+            # Env vars for osmcha database
             {{ if eq $job.name "osmcha-db" }}
               # DB credentials
               - name: POSTGRES_HOST

osm-seed/templates/letsencrypt-issuer.yaml

@@ -1,4 +1,4 @@
-{{- if eq .Values.serviceType "ClusterIP" }}
+{{- if and (eq .Values.serviceType "ClusterIP") (eq .Values.createClusterIssuer true) }}
 apiVersion: cert-manager.io/v1
 kind: ClusterIssuer
 metadata:

osm-seed/templates/tasking-manager-api/tm-deployment.yaml

@@ -46,17 +46,17 @@ spec:
           {{- end }}
           env:
             - name: POSTGRES_HOST
-              value: {{ .Values.tmApi.env.POSTGRES_HOST }}
+              value: {{ .Release.Name }}-tm-db
             - name: POSTGRES_DB
-              value: {{ .Values.tmApi.env.POSTGRES_DB }}
+              value: {{ .Values.tmDb.env.POSTGRES_DB }}
             - name: POSTGRES_PASSWORD
-              value: {{ quote .Values.tmApi.env.POSTGRES_PASSWORD }}
+              value: {{ quote .Values.tmDb.env.POSTGRES_PASSWORD }}
             - name: POSTGRES_USER
-              value: {{ .Values.tmApi.env.POSTGRES_USER }}
+              value: {{ .Values.tmDb.env.POSTGRES_USER }}
             - name: POSTGRES_PORT
-              value: {{ .Values.tmApi.env.POSTGRES_PORT | quote }}
+              value: "5432"
             - name: TM_DB
-              value: {{ printf "postgresql://%s:%s@%s/%s" .Values.tmApi.env.POSTGRES_USER .Values.tmApi.env.POSTGRES_PASSWORD .Values.tmApi.env.POSTGRES_HOST .Values.tmApi.env.POSTGRES_DB }}
+              value: {{ printf "postgresql://%s:%s@%s-tm-db/%s" .Values.tmDb.env.POSTGRES_USER .Values.tmDb.env.POSTGRES_PASSWORD .Release.Name .Values.tmDb.env.POSTGRES_DB }}
             - name: TM_ORG_NAME
               value: {{ .Values.tmApi.env.TM_ORG_NAME }}
             - name: TM_ORG_CODE

osm-seed/templates/tasking-manager-db/tm-db-pd.yaml

@@ -0,0 +1,85 @@
+{{- if .Values.tmDb.enabled -}}
+{{- if .Values.tmDb.persistenceDisk.enabled -}}
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: {{ .Release.Name }}-tm-db-pv
+  labels:
+    app: {{ template "osm-seed.name" . }}
+    component: tm-db-pd
+    environment: {{ .Values.environment }}
+    release: {{ .Release.Name }}
+spec:
+  ############## Minikube ##############
+  {{- if eq .Values.cloudProvider "minikube" }}
+  storageClassName: local-storage
+  capacity:
+    storage: {{ .Values.tmDb.persistenceDisk.localVolumeSize }}
+  accessModes:
+    - ReadWriteOnce
+  hostPath:
+    path: {{ .Values.tmDb.persistenceDisk.localVolumeHostPath | quote }}
+  {{- end }}
+
+  ############## AWS Provider ##############
+  {{- if eq .Values.cloudProvider "aws" }}
+  storageClassName: ""
+  capacity:
+    storage: {{ .Values.tmDb.persistenceDisk.AWS_ElasticBlockStore_size }}
+  accessModes:
+    - ReadWriteOnce
+  awsElasticBlockStore:
+    volumeID: {{ .Values.tmDb.persistenceDisk.AWS_ElasticBlockStore_volumeID }}
+    fsType: ext4
+  {{- end }}
+
+  ############## GCP Provider ##############
+  {{- if eq .Values.cloudProvider "gcp" }}
+  storageClassName: ""
+  capacity:
+    storage: {{ .Values.tmDb.persistenceDisk.GCP_gcePersistentDisk_size }}
+  accessModes:
+    - ReadWriteOnce
+  gcePersistentDisk:
+    pdName: {{ .Values.tmDb.persistenceDisk.GCP_gcePersistentDisk_pdName }}
+    fsType: ext4
+  {{- end }}
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: {{ .Release.Name }}-tm-db-pv-claim
+  labels:
+    app: {{ template "osm-seed.name" . }}
+    component: db-pd
+    environment: {{ .Values.environment }}
+    release: {{ .Release.Name }}
+spec:
+  ############## Minikube ##############
+  {{- if eq .Values.cloudProvider "minikube" }}
+  storageClassName: local-storage
+  {{- else }}
+  ############## Other providers ##############
+  storageClassName: ""
+  {{- end }}
+  volumeName: {{ .Release.Name }}-tm-db-pv
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      ############# Minikube ##############
+      {{- if eq .Values.cloudProvider "minikube" }}
+      storage: {{ .Values.tmDb.persistenceDisk.localVolumeSize }}
+      {{- end }}
+
+      ############## AWS Provider ##############
+      {{- if eq .Values.cloudProvider "aws" }}
+      storage: {{ .Values.tmDb.persistenceDisk.AWS_ElasticBlockStore_size }}
+      {{- end }}
+
+      ############## GCP Provider ##############
+      {{- if eq .Values.cloudProvider "gcp" }}
+      storage: {{ .Values.tmDb.persistenceDisk.GCP_gcePersistentDisk_size }}
+      {{- end }}
+{{- end }}
+{{- end }}

osm-seed/templates/tasking-manager-db/tm-db-service.yaml

@@ -0,0 +1,19 @@
+{{- if .Values.tmDb.enabled -}}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Release.Name }}-tm-db
+  labels:
+    app: {{ template "osm-seed.name" . }}
+    component: tm-db-service
+    environment: {{ .Values.environment }}
+    release: {{ .Release.Name }}
+spec:
+  ports:
+    - port: 5432
+      targetPort: 5432
+  selector:
+    app: {{ template "osm-seed.name" . }}
+    release: {{ .Release.Name }}
+    run: {{ .Release.Name }}-tm-db
+{{- end }}

osm-seed/templates/tasking-manager-db/tm-db-statefulset.yaml

@@ -0,0 +1,101 @@
+{{- if .Values.tmDb.enabled -}}
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{ .Release.Name }}-tm-db
+  labels:
+    app: {{ template "osm-seed.name" . }}
+    component: db-tm-statefulset
+    environment: {{ .Values.environment }}
+    release: {{ .Release.Name }}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: {{ template "osm-seed.name" . }}
+      release: {{ .Release.Name }}
+      run: {{ .Release.Name }}-tm-db
+  serviceName: {{ .Release.Name }}-tm-db
+  template:
+    metadata:
+      labels:
+        app: {{ template "osm-seed.name" . }}
+        release: {{ .Release.Name }}
+        run: {{ .Release.Name }}-tm-db
+    spec:
+      containers:
+        - name: {{ .Chart.Name }}-tm-db
+          image: "{{ .Values.tmDb.image.name }}:{{ .Values.tmDb.image.tag }}"
+          ports:
+            - name: postgres
+              containerPort: 5432
+              protocol: TCP
+          env:
+            - name: POSTGRES_HOST
+              value: {{ .Release.Name }}-tm-db
+            - name: POSTGRES_DB
+              value: {{ .Values.tmDb.env.POSTGRES_DB }}
+            - name: POSTGRES_PASSWORD
+              value: {{ quote .Values.tmDb.env.POSTGRES_PASSWORD }}
+            - name: POSTGRES_USER
+              value: {{ default "postgres" .Values.tmDb.env.POSTGRES_USER | quote }}
+            - name: PGUSER
+              value: {{ default "postgres" .Values.tmDb.env.POSTGRES_USER | quote }}
+            - name: PGDATA
+              value: {{ .Values.tmDb.persistenceDisk.mountPath }}
+            - name: POSTGRES_DB_MAX_CONNECTIONS
+              value: {{ default 100 .Values.tmDb.env.POSTGRES_DB_MAX_CONNECTIONS | quote }}
+            - name: POSTGRES_DB_SHARED_BUFFERS
+              value: {{ default "128MB" .Values.tmDb.env.POSTGRES_DB_SHARED_BUFFERS | quote }}
+            - name: POSTGRES_DB_WORK_MEM
+              value: {{ default "4MB" .Values.tmDb.env.POSTGRES_DB_WORK_MEM | quote }}
+            - name: POSTGRES_DB_MAINTENANCE_WORK_MEM
+              value: {{ default "64MB" .Values.tmDb.env.POSTGRES_DB_MAINTENANCE_WORK_MEM | quote }}
+            - name: POSTGRES_DB_EFFECTIVE_CACHE_SIZE
+              value: {{ default "4GB" .Values.tmDb.env.POSTGRES_DB_EFFECTIVE_CACHE_SIZE | quote }}
+            - name: POD_IP
+              valueFrom: { fieldRef: { fieldPath: status.podIP } }
+          livenessProbe:
+            exec:
+              command:
+              - sh
+              - -c
+              - exec pg_isready --host $POD_IP
+            initialDelaySeconds: 120
+            timeoutSeconds: 5
+            failureThreshold: 6
+          readinessProbe:
+            exec:
+              command:
+              - sh
+              - -c
+              - exec pg_isready --host $POD_IP
+            initialDelaySeconds: 5
+            timeoutSeconds: 3
+            periodSeconds: 5
+          volumeMounts:
+            - name: postgres-storage
+              mountPath: {{ .Values.tmDb.persistenceDisk.mountPath }}
+              subPath: {{ .Values.tmDb.persistenceDisk.subPath }}
+          {{- if .Values.tmDb.resources.enabled }}
+          resources:
+            requests:
+              memory: {{ .Values.tmDb.resources.requests.memory }}
+              cpu: {{ .Values.tmDb.resources.requests.cpu }}
+            limits:
+              memory: {{ .Values.tmDb.resources.limits.memory }}
+              cpu: {{ .Values.tmDb.resources.limits.cpu }}
+          {{- end }}
+      volumes:
+        - name: postgres-storage
+        {{- if .Values.tmDb.persistenceDisk.enabled }}
+          persistentVolumeClaim:
+            claimName: {{ .Release.Name }}-tm-db-pv-claim
+        {{- else }}
+          emptyDir: {}
+        {{- end }}
+      {{- if .Values.tmDb.nodeSelector.enabled }}
+      nodeSelector:
+        {{ .Values.tmDb.nodeSelector.label_key }} : {{ .Values.tmDb.nodeSelector.label_value }}
+      {{- end }}
+{{- end }}

osm-seed/values.yaml

@@ -41,6 +41,7 @@ AWS_SSL_ARN: false
 # SSL separately.
 # serviceType: NodePort
 serviceType: clusterIP
+createClusterIssuer: true
 # Domain that is pointed to the clusterIP
 # You will need to create an A record like *.osmseed.example.com pointed to the ClusterIP
 # Then, the cluster configuration will setup services at their respective subdomains:
@@ -73,9 +74,9 @@ db:
   env:
     POSTGRES_DB: openstreetmap
     POSTGRES_USER: postgres
-    POSTGRES_PASSWORD: 1234
+    POSTGRES_PASSWORD: "1234"
+    ENVIRONMENT: "production"
     LOG_STATEMENT: "all"
-    # LOG_STATEMENT=all, ddl, mod
   persistenceDisk:
     enabled: false
     mountPath: /var/lib/postgresql/data
@@ -513,6 +514,30 @@ tilerVisor:
     label_key: nodegroup_type
     label_value: tiler
 # ====================================================================================================
+# Variables for Tasking Manager DB
+# ====================================================================================================
+tmDb:
+  enabled: false
+  image:
+    name: "postgres"
+    tag: "11"
+  nodeSelector:
+    enabled: false
+    label_key: nodegroup_type
+    label_value: web
+  env:
+    POSTGRES_DB: tm
+    POSTGRES_PASSWORD: tm
+    POSTGRES_USER: postgres
+  resources:
+    enabled: false
+    requests:
+      memory: "1Gi"
+      cpu: "2"
+    limits:
+      memory: "2Gi"
+      cpu: "2"
+# ====================================================================================================
 # Variables for Tasking Manager API
 # ====================================================================================================