Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add configMap for tiler #351

Merged
merged 10 commits into from
Nov 29, 2024
Merged

Add configMap for tiler #351

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
10eca67
Increase initialDelaySeconds for tiler db deployment
Rub21 Nov 15, 2024
8669f20
Add configmap for tiler server
Rub21 Nov 15, 2024
3922b6b
Add service for prometheus - tiler server
Rub21 Nov 15, 2024
2906caa
Remove service for prometheus - tiler server
Rub21 Nov 15, 2024
94ad0fe
Update tiler db templates - adding sharedMemorySize
Rub21 Nov 18, 2024
37a0f93
Set default sharedMemorySize
Rub21 Nov 18, 2024
6779f4a
Add ttlSecondsAfterFinished for cronjob - osmcha
Rub21 Nov 26, 2024
5f7eff5
Add env var for imposm - tiler
Rub21 Nov 28, 2024
b7799d4
Add env vars for tiler server
Rub21 Nov 29, 2024
370bddd
Add default values for EXECUTE_REINDEX and EXECUTE_VACUUM_ANALYZE
Rub21 Nov 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 16 additions & 8 deletions osm-seed/templates/osmcha-app/cronJob.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,11 @@ metadata:
spec:
schedule: "*/2 * * * *"
concurrencyPolicy: Forbid
successfulJobsHistoryLimit: 2
failedJobsHistoryLimit: 2
successfulJobsHistoryLimit: 1
failedJobsHistoryLimit: 1
jobTemplate:
spec:
ttlSecondsAfterFinished: 120
template:
spec:
containers:
Expand All @@ -32,12 +33,12 @@ spec:
restartPolicy: Never
{{- if .Values.osmchaApi.nodeSelector.enabled }}
nodeSelector:
{{ .Values.osmchaApi.nodeSelector.label_key }} : {{ .Values.osmchaApi.nodeSelector.label_value }}
{{ .Values.osmchaApi.nodeSelector.label_key }}: {{ .Values.osmchaApi.nodeSelector.label_value }}
{{- end }}

---
apiVersion: batch/v1
kind: CronJob
kind: CronJob
metadata:
name: {{ .Release.Name }}-osmcha-process-changesets-cronjob
labels:
Expand All @@ -48,10 +49,11 @@ metadata:
spec:
schedule: "0 * * * *"
concurrencyPolicy: Forbid
successfulJobsHistoryLimit: 2
failedJobsHistoryLimit: 2
successfulJobsHistoryLimit: 1
failedJobsHistoryLimit: 1
jobTemplate:
spec:
ttlSecondsAfterFinished: 120
template:
spec:
containers:
Expand All @@ -62,13 +64,19 @@ spec:
- /bin/sh
- -c
- |
python manage.py backfill_changesets $(date -d "yesterday" +%Y-%m-%d) $(date +%Y-%m-%d)
YESTERDAY=$(date -d "yesterday" +%Y-%m-%d)
TODAY=$(date +%Y-%m-%d)

python manage.py backfill_changesets "$YESTERDAY" "$TODAY"
EXIT_CODE=$?

exit $EXIT_CODE
envFrom:
- configMapRef:
name: {{ .Release.Name }}-osmcha-common-env
restartPolicy: Never
{{- if .Values.osmchaApi.nodeSelector.enabled }}
nodeSelector:
{{ .Values.osmchaApi.nodeSelector.label_key }} : {{ .Values.osmchaApi.nodeSelector.label_value }}
{{ .Values.osmchaApi.nodeSelector.label_key }}: {{ .Values.osmchaApi.nodeSelector.label_value }}
{{- end }}
{{- end }}
6 changes: 6 additions & 0 deletions osm-seed/templates/tiler-db/tiler-db-statefulset.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,8 @@ spec:
- name: tiler-db-postgresql-config
mountPath: /etc/postgresql/
{{- end }}
- name: shared-memory
mountPath: /dev/shm
{{- if .Values.tilerDb.resources.enabled }}
resources:
requests:
Expand All @@ -101,6 +103,10 @@ spec:
configMap:
name: {{ .Release.Name }}-tiler-db-postgresql-config
{{- end }}
- name: shared-memory
emptyDir:
medium: Memory
sizeLimit: {{ .Values.tilerDb.sharedMemorySize | default "64Mi" }}
{{- if .Values.tilerDb.nodeSelector.enabled }}
nodeSelector:
{{ .Values.tilerDb.nodeSelector.label_key }} : {{ .Values.tilerDb.nodeSelector.label_value }}
Expand Down
2 changes: 2 additions & 0 deletions osm-seed/templates/tiler-imposm/tiler-imposm-statefulset.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,8 @@ spec:
value: {{ .Values.tilerImposm.env.IMPORT_NATURAL_EARTH | default "true" | quote }}
- name: IMPORT_OSM_LAND
value: {{ .Values.tilerImposm.env.IMPORT_OSM_LAND | default "true" | quote }}
- name: IMPOSM3_IMPORT_LAYERS
value: {{ .Values.tilerImposm.env.IMPOSM3_IMPORT_LAYERS | default "all" | quote }}
- name: CLOUDPROVIDER
value: {{ .Values.cloudProvider }}
# In case cloudProvider=aws
Expand Down
50 changes: 3 additions & 47 deletions osm-seed/templates/tiler-server/tiler-server-cache-cleaner-deployment.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,53 +33,9 @@ spec:
memory: {{ .Values.tilerServerCacheCleaner.resources.limits.memory }}
cpu: {{ .Values.tilerServerCacheCleaner.resources.limits.cpu }}
{{- end }}
env:
# tiler-db env variables
- name: POSTGRES_HOST
value: {{ .Release.Name }}-tiler-db
- name: POSTGRES_DB
value: {{ .Values.tilerDb.env.POSTGRES_DB }}
- name: POSTGRES_PASSWORD
value: {{ quote .Values.tilerDb.env.POSTGRES_PASSWORD }}
- name: POSTGRES_USER
value: {{ .Values.tilerDb.env.POSTGRES_USER }}
- name: POSTGRES_PORT
value: {{ .Values.tilerDb.env.POSTGRES_PORT | quote }}
# tiler-server env variables
- name: TILER_SERVER_PORT
value: {{ .Values.tilerServer.env.TILER_SERVER_PORT | quote }}
- name: TILER_CACHE_TYPE
value: {{ .Values.tilerServer.env.TILER_CACHE_TYPE }}
- name: TILER_CACHE_REGION
value: {{ .Values.tilerServer.env.TILER_CACHE_REGION }}
- name: TILER_CACHE_BUCKET
value: {{ .Values.tilerServer.env.TILER_CACHE_BUCKET }}
- name: TILER_CACHE_BASEPATH
value: {{ quote .Values.tilerServer.env.TILER_CACHE_BASEPATH }}
- name: TILER_CACHE_AWS_ACCESS_KEY_ID
value: {{ quote .Values.tilerServer.env.TILER_CACHE_AWS_ACCESS_KEY_ID }}
- name: TILER_CACHE_AWS_SECRET_ACCESS_KEY
value: {{ quote .Values.tilerServer.env.TILER_CACHE_AWS_SECRET_ACCESS_KEY }}
- name: TILER_CACHE_MAX_ZOOM
value: {{ quote .Values.tilerServer.env.TILER_CACHE_MAX_ZOOM }}
- name: MAX_NUM_PS
value: {{ quote .Values.tilerServerCacheCleaner.env.MAX_NUM_PS }}
- name: PROCESS_NAME
value: {{ .Values.tilerServerCacheCleaner.env.PROCESS_NAME }}
- name: KILL_PROCESS
value: {{ .Values.tilerServerCacheCleaner.env.KILL_PROCESS }}
- name: CLOUDPROVIDER
value: {{ .Values.cloudProvider }}
# In case cloudProvider=aws
{{- if eq .Values.cloudProvider "aws" }}
- name: AWS_S3_BUCKET
value: {{ .Values.AWS_S3_BUCKET }}
{{- end }}
# In case cloudProvider=gcp
{{- if eq .Values.cloudProvider "gcp" }}
- name: GCP_STORAGE_BUCKET
value: {{ .Values.GCP_STORAGE_BUCKET }}
{{- end }}
envFrom:
- configMapRef:
name: {{ .Release.Name }}-tiler-server-cm
{{- if .Values.tilerServerCacheCleaner.nodeSelector.enabled }}
nodeSelector:
{{ .Values.tilerServerCacheCleaner.nodeSelector.label_key }} : {{ .Values.tilerServerCacheCleaner.nodeSelector.label_value }}
Expand Down
29 changes: 29 additions & 0 deletions osm-seed/templates/tiler-server/tiler-server-configMap.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
{{- if .Values.tilerServer.enabled -}}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ .Release.Name }}-tiler-server-cm
data:
POSTGRES_HOST: {{ .Release.Name }}-tiler-db
POSTGRES_DB: {{ .Values.tilerDb.env.POSTGRES_DB | quote }}
POSTGRES_PASSWORD: {{ .Values.tilerDb.env.POSTGRES_PASSWORD | quote }}
POSTGRES_USER: {{ .Values.tilerDb.env.POSTGRES_USER | quote }}
POSTGRES_PORT: {{ .Values.tilerDb.env.POSTGRES_PORT | quote }}
TILER_SERVER_PORT: {{ .Values.tilerServer.env.TILER_SERVER_PORT | quote }}
TILER_CACHE_TYPE: {{ .Values.tilerServer.env.TILER_CACHE_TYPE | quote }}
TILER_CACHE_REGION: {{ .Values.tilerServer.env.TILER_CACHE_REGION | quote }}
TILER_CACHE_BUCKET: {{ .Values.tilerServer.env.TILER_CACHE_BUCKET | quote }}
TILER_CACHE_BASEPATH: {{ .Values.tilerServer.env.TILER_CACHE_BASEPATH | quote }}
TILER_CACHE_AWS_ACCESS_KEY_ID: {{ .Values.tilerServer.env.TILER_CACHE_AWS_ACCESS_KEY_ID | quote }}
TILER_CACHE_AWS_SECRET_ACCESS_KEY: {{ .Values.tilerServer.env.TILER_CACHE_AWS_SECRET_ACCESS_KEY | quote }}
TILER_CACHE_MAX_ZOOM: {{ .Values.tilerServer.env.TILER_CACHE_MAX_ZOOM | quote }}
EXECUTE_REINDEX: {{ .Values.tilerServer.env.EXECUTE_REINDEX | default "false" | quote }}
EXECUTE_VACUUM_ANALYZE: {{ .Values.tilerServer.env.EXECUTE_VACUUM_ANALYZE | default "false" | quote }}
CLOUDPROVIDER: {{ .Values.cloudProvider | quote }}
{{- if eq .Values.cloudProvider "aws" }}
AWS_S3_BUCKET: {{ .Values.AWS_S3_BUCKET | quote }}
{{- end }}
{{- if eq .Values.cloudProvider "gcp" }}
GCP_STORAGE_BUCKET: {{ .Values.GCP_STORAGE_BUCKET | quote }}
{{- end }}
{{- end }}
46 changes: 4 additions & 42 deletions osm-seed/templates/tiler-server/tiler-server-deployment.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ spec:
httpGet:
path: /
port: {{ .Values.tilerServer.env.TILER_SERVER_PORT }}
initialDelaySeconds: 600 # 10 min, because the compile process takes time.
initialDelaySeconds: 1500
timeoutSeconds: 30
{{- if .Values.tilerServer.resources.enabled }}
resources:
Expand All @@ -46,47 +46,9 @@ spec:
memory: {{ .Values.tilerServer.resources.limits.memory }}
cpu: {{ .Values.tilerServer.resources.limits.cpu }}
{{- end }}
env:
# tiler-db env variables
- name: POSTGRES_HOST
value: {{ .Release.Name }}-tiler-db
- name: POSTGRES_DB
value: {{ .Values.tilerDb.env.POSTGRES_DB }}
- name: POSTGRES_PASSWORD
value: {{ quote .Values.tilerDb.env.POSTGRES_PASSWORD }}
- name: POSTGRES_USER
value: {{ .Values.tilerDb.env.POSTGRES_USER }}
- name: POSTGRES_PORT
value: {{ .Values.tilerDb.env.POSTGRES_PORT | quote }}
# tiler-server env variables
- name: TILER_SERVER_PORT
value: {{ .Values.tilerServer.env.TILER_SERVER_PORT | quote }}
- name: TILER_CACHE_TYPE
value: {{ .Values.tilerServer.env.TILER_CACHE_TYPE }}
- name: TILER_CACHE_REGION
value: {{ .Values.tilerServer.env.TILER_CACHE_REGION }}
- name: TILER_CACHE_BUCKET
value: {{ .Values.tilerServer.env.TILER_CACHE_BUCKET }}
- name: TILER_CACHE_BASEPATH
value: {{ quote .Values.tilerServer.env.TILER_CACHE_BASEPATH }}
- name: TILER_CACHE_AWS_ACCESS_KEY_ID
value: {{ quote .Values.tilerServer.env.TILER_CACHE_AWS_ACCESS_KEY_ID }}
- name: TILER_CACHE_AWS_SECRET_ACCESS_KEY
value: {{ quote .Values.tilerServer.env.TILER_CACHE_AWS_SECRET_ACCESS_KEY }}
- name: TILER_CACHE_MAX_ZOOM
value: {{ quote .Values.tilerServer.env.TILER_CACHE_MAX_ZOOM }}
- name: CLOUDPROVIDER
value: {{ .Values.cloudProvider }}
# In case cloudProvider=aws
{{- if eq .Values.cloudProvider "aws" }}
- name: AWS_S3_BUCKET
value: {{ .Values.AWS_S3_BUCKET }}
{{- end }}
# In case cloudProvider=gcp
{{- if eq .Values.cloudProvider "gcp" }}
- name: GCP_STORAGE_BUCKET
value: {{ .Values.GCP_STORAGE_BUCKET }}
{{- end }}
envFrom:
- configMapRef:
name: {{ .Release.Name }}-tiler-server-cm
{{- if .Values.tilerServer.nodeSelector.enabled }}
nodeSelector:
{{ .Values.tilerServer.nodeSelector.label_key }} : {{ .Values.tilerServer.nodeSelector.label_value }}
Expand Down
19 changes: 19 additions & 0 deletions osm-seed/values.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -376,6 +376,7 @@ tilerDb:
# GCP
GCP_gcePersistentDisk_pdName: osmseed-disk-tilerdb-v1
GCP_gcePersistentDisk_size: 50Gi
sharedMemorySize: 64Mi
resources:
enabled: false
requests:
Expand Down Expand Up @@ -423,6 +424,7 @@ tilerImposm:
UPLOAD_EXPIRED_FILES: true
IMPORT_NATURAL_EARTH: true
IMPORT_OSM_LAND: true
IMPOSM3_IMPORT_LAYERS: all
persistenceDisk:
enabled: false
accessMode: ReadWriteOnce
Expand Down Expand Up @@ -469,6 +471,8 @@ tilerServer:
TILER_CACHE_REGION: us-east-1
TILER_CACHE_AWS_ACCESS_KEY_ID: xyz
TILER_CACHE_AWS_SECRET_ACCESS_KEY: xyz/q
EXECUTE_REINDEX: false
EXECUTE_VACUUM_ANALYZE: false
command: "./start.sh" # ./start.sh & ./tile_cache_downloader.sh & ./expire-watcher.sh
# In case you use TILER_CACHE_TYPE: file with persistenceDisk
persistenceDisk:
Expand Down Expand Up @@ -526,6 +530,21 @@ tilerServerCacheCleaner:
minReplicas: 1
maxReplicas: 2
cpuUtilization: 60

tilerCacheCleanerJob:
enabled: false
tilesFile: imposm/imposm3_expire_dir/latest.tiles
minZoom: 0
maxZoom: 20
overwrite: true
resources:
enabled: false
requests:
memory: "1Gi"
cpu: "500m"
limits:
memory: "2Gi"
cpu: "1"
# ====================================================================================================
# Variables for Tasking Manager DB
# ====================================================================================================
Expand Down
Loading