s3sync-cronjob.template.yaml

---
kind: Template
apiVersion: template.openshift.io/v1
metadata:
  name: tangerine-backend-s3sync

parameters:
  - name: AWS_S3_SECRET_NAME
    value: s3-credentials
  - name: S3_SYNC_SUSPEND
    value: "false"
  - name: EMBED_BASE_URL
    value: http://text-embeddings-inference:3000/v1
  - name: EMBED_MODEL_NAME
    value: Snowflake/snowflake-arctic-embed-m-long
  - name: EMBED_QUERY_PREFIX
    value: "Represent this sentence for searching relevant passages"
  - name: CPU_LIMIT
    value: 1000m
  - name: MEMORY_LIMIT
    value: 1Gi
  - name: CPU_REQUEST
    value: 1000m
  - name: MEMORY_REQUEST
    value: 1Gi
  - name: IMAGE
    value: quay.io/tangerine/tangerine-backend
  - name: IMAGE_TAG
    value: latest
  - name: FORCE_RESYNC_UNTIL
    value: '0001-01-01T00:00:00Z'  # default is "the start of time", it will never run

objects:
- kind: Secret
  apiVersion: v1
  metadata:
    name: s3-credentials
  stringData:
    aws_access_key_id: your_key_id
    aws_secret_access_key: your_access_key
    aws_region: your_aws_region
  type: Opaque

- apiVersion: v1
  kind: ConfigMap
  metadata:
    name: tangerine-s3sync
  data:
    # example config
    s3.yaml: |
      defaults:
        extensions:
          - html
          - md
        citation_url_template: 'https://files.test/{{ full_path }}'

      agents:
        - name: agent1
          description: Agent One
          bucket: mybucket
          paths:
            - prefix: path/in/bucket

        - name: agent2
          description: Agent Two
          bucket: mybucket
          paths:
            - prefix: other/path/in/bucket
              citation_url_template: 'https://some.site/{{ full_path | replace("other/", "") }}'

- kind: CronJob
  apiVersion: batch/v1
  metadata:
    name: tangerine-s3-sync
  spec:
    suspend: ${{S3_SYNC_SUSPEND}}
    schedule: "*/30 * * * *"
    concurrencyPolicy: "Replace"
    startingDeadlineSeconds: 1200
    jobTemplate:
      spec:
        template:
          spec:
            volumes:
              - name: s3sync-config
                configMap:
                  name: tangerine-s3sync
            containers:
              - name: tangerine-s3sync
                command: ["flask", "s3sync"]
                volumeMounts:
                  - name: s3sync-config
                    mountPath: /s3sync-config
                resources:
                  requests:
                    memory: ${MEMORY_REQUEST}
                    cpu: ${CPU_REQUEST}
                  limits:
                    memory: ${MEMORY_LIMIT}
                    cpu: ${CPU_LIMIT}
                env:
                  - name: S3_SYNC_CONFIG_FILE
                    value: /s3sync-config/s3.yaml
                  - name: AWS_ACCESS_KEY_ID
                    valueFrom:
                      secretKeyRef:
                        name: ${AWS_S3_SECRET_NAME}
                        key: aws_access_key_id
                  - name: AWS_SECRET_ACCESS_KEY
                    valueFrom:
                      secretKeyRef:
                        name: ${AWS_S3_SECRET_NAME}
                        key: aws_secret_access_key
                  - name: AWS_DEFAULT_REGION
                    valueFrom:
                      secretKeyRef:
                        name: ${AWS_S3_SECRET_NAME}
                        key: aws_region
                  - name: DB_USERNAME
                    valueFrom:
                      secretKeyRef:
                        name: vector-db
                        key: db.user
                  - name: DB_PASSWORD
                    valueFrom:
                      secretKeyRef:
                        name: vector-db
                        key: db.password
                  - name: DB_NAME
                    valueFrom:
                      secretKeyRef:
                        name: vector-db
                        key: db.name
                  - name: DB_HOST
                    valueFrom:
                      secretKeyRef:
                        name: vector-db
                        key: db.host
                  - name: DB_PORT
                    valueFrom:
                      secretKeyRef:
                        name: vector-db
                        key: db.port
                  - name: EMBED_BASE_URL
                    value: ${EMBED_BASE_URL}
                  - name: EMBED_MODEL_NAME
                    value: ${EMBED_MODEL_NAME}
                  - name: EMBED_QUERY_PREFIX
                    value: ${EMBED_QUERY_PREFIX}
                  - name: EMBED_API_KEY
                    valueFrom:
                      secretKeyRef:
                        name: embed-api-key
                        key: api_key
                  - name: FORCE_RESYNC_UNTIL
                    value: ${FORCE_RESYNC_UNTIL}
                imagePullPolicy: IfNotPresent
                image: ${IMAGE}:${IMAGE_TAG}
            restartPolicy: Never