docker-compose.yml

volumes:
  postgres-data:
    driver: local
  kestra-data:
    driver: local

services:
  postgres:
    image: postgres
    pull_policy: missing
    volumes:
      - postgres-data:/var/lib/postgresql/data
    environment:
      POSTGRES_DB: kestra
      POSTGRES_USER: kestra
      POSTGRES_PASSWORD: k3str4
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -d $${POSTGRES_DB} -U $${POSTGRES_USER}"]
      interval: 30s
      timeout: 10s
      retries: 10
    ports:
      - "5432:5432"

  kestra:
    image: kestra/kestra:latest # during development, you can use kestra/kestra:develop
    pull_policy: always # missing
    entrypoint: /bin/bash
    user: "root"
    env_file:
      - .env
      - .env_encoded
    command:
      - -c
      - /app/kestra server standalone --worker-thread=128 # && /app/kestra plugins install custom_plugins/
    volumes:
      - /Users/anna/.kestra/storage:/app/storage
      - /var/run/docker.sock:/var/run/docker.sock
      - /tmp/kestra-wd:/tmp/kestra-wd:rw
#      - /Users/anna/dev/plugins:/app/custom_plugins
    environment:
      JAVA_OPTS: "--add-opens java.base/java.nio=ALL-UNNAMED"
      NODE_OPTIONS: "--max-old-space-size=4096"
      KESTRA_CONFIGURATION: |
        datasources:
          postgres:
            url: jdbc:postgresql://postgres:5432/kestra
            driverClassName: org.postgresql.Driver
            username: kestra
            password: k3str4
        kestra:
          server:
            basic-auth:
              enabled: false
              username: admin
              password: kestra
          anonymous-usage-report:
            enabled: false
          repository:
            type: postgres
          storage:
            type: local
            local:
              base-path: "/app/storage"
          queue:
            type: postgres
          tasks:
            tmp-dir:
              path: /tmp/kestra-wd/tmp
            scripts:
              docker:
                volume-enabled: true
            defaults:
              - type: io.kestra.plugin.airbyte.connections.Sync
                values:
                  url: http://host.docker.internal:8000/
                  username: "{{ secret('AIRBYTE_USERNAME') }}"
                  password: "{{ secret('AIRBYTE_PASSWORD') }}"
              - type: io.kestra.plugin.aws.s3.Upload
                values:
                  accessKeyId: "{{ secret('AWS_ACCESS_KEY_ID') }}"
                  secretKeyId: "{{ secret('AWS_SECRET_ACCESS_KEY') }}"
                  region: "{{ secret('AWS_DEFAULT_REGION') }}"
              - type: io.kestra.plugin.aws.s3.Download
                values:
                  accessKeyId: "{{ secret('AWS_ACCESS_KEY_ID') }}"
                  secretKeyId: "{{ secret('AWS_SECRET_ACCESS_KEY') }}"
                  region: "{{ secret('AWS_DEFAULT_REGION') }}"
              - type: io.kestra.plugin.aws.cli.AwsCLI
                values:
                  accessKeyId: "{{ secret('AWS_ACCESS_KEY_ID') }}"
                  secretKeyId: "{{ secret('AWS_SECRET_ACCESS_KEY') }}"
                  region: "{{ secret('AWS_DEFAULT_REGION') }}"
              - type: io.kestra.plugin.aws.athena.Query
                values:
                  accessKeyId: "{{ secret('AWS_ACCESS_KEY_ID') }}"
                  secretKeyId: "{{ secret('AWS_SECRET_ACCESS_KEY') }}"
                  region: "{{ secret('AWS_DEFAULT_REGION') }}"
                  outputLocation: "s3://kestraio/query_results/"
          url: http://localhost:8080/
          variables:
            env-vars-prefix: ""
    ports:
      - "8081:8081"
      - "8080:8080"
    depends_on:
      postgres:
        condition: service_started