Update config, remove deployment, add revoke cronjob

ingest/Snakefile

@@ -446,8 +446,8 @@ rule approve:
         log_level=LOG_LEVEL,
     shell:
         """
-        python {input.script} \
+        timeout 25m bash -c "python {input.script} \
             --mode approve \
             --config-file {input.config} \
-            --log-level {params.log_level} \
+            --log-level {params.log_level}" \
         """

ingest/scripts/prepare_files.py

@@ -13,6 +13,7 @@
 
 @dataclass
 class Config:
+    organism: str
     segmented: str
     nucleotide_sequences: list[str]
     slack_hook: str
@@ -52,8 +53,8 @@ def revocation_notification(config: Config, to_revoke: dict[str, dict[str, str]]
     text = (
         f"{config.backend_url}: Ingest pipeline wants to add the following sequences"
         f" which will lead to revocations: {to_revoke}. "
-        "If you agree with this run the regroup_and_revoke rule in the ingest pod:"
-        " `kubectl exec -it INGEST_POD_NAME -- snakemake regroup_and_revoke`."
+        "If you agree with this manually run the regroup_and_revoke cronjob:"
+        f" `kubectl create job --from=cronjob/loculus-revoke-and-regroup-cronjob-{config.organism} <manual-job-name>`."
     )
     notify(config, text)
 

kubernetes/loculus/templates/ingest-deployment.yaml

@@ -3,74 +3,67 @@
 {{- range $key, $value := (.Values.organisms | default .Values.defaultOrganisms) }}
 {{- if $value.ingest }}
 ---
-apiVersion: apps/v1
-kind: Deployment
+apiVersion: batch/v1
+kind: CronJob
 metadata:
-  name: loculus-ingest-{{ $key }}
-  annotations:
-    argocd.argoproj.io/sync-options: Replace=true
-    reloader.stakater.com/auto: "true"
+  name: loculus-ingest-cronjob-{{ $key }}
 spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: loculus
-      component: loculus-ingest-{{ $key }}
-  template:
-    metadata:
-      labels:
-        app: loculus
-        component: loculus-ingest-{{ $key }}
+  schedule: "*/2 * * * *" # ingest every 2 minutes but forbid concurrency, have jobs run only for Values.ingestLimitSeconds
+  startingDeadlineSeconds: 60
+  concurrencyPolicy: Forbid
+  jobTemplate:
     spec:
-      containers:
-        - name: ingest-{{ $key }}
-          image: {{ $value.ingest.image}}:{{ $dockerTag }}
-          imagePullPolicy: Always
-          resources:
-            requests:
-              memory: "80Mi"
-              cpu: "10m"
-            limits:
-              memory: "10Gi"
-          env:
-            - name: KEYCLOAK_INGEST_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: service-accounts
-                  key: insdcIngestUserPassword
-            - name: NCBI_API_KEY
-              valueFrom:
-                secretKeyRef:
-                  name: ingest-ncbi
-                  key: api-key
-            - name: SLACK_HOOK
-              valueFrom:
-                secretKeyRef:
-                  name: slack-notifications
-                  key: slack-hook
-          args:
-            - snakemake
-            - results/approved
-            - results/submitted # Remove in production, see #1777
-            - results/revised # Remove in production, see #1777
-            - --all-temp # Reduce disk usage by not keeping files around
-      {{- if $value.ingest.configFile }}
-          volumeMounts:
+      activeDeadlineSeconds: {{ $.Values.ingestLimitSeconds }}
+      template:
+        metadata:
+          labels:
+            app: loculus
+            component: loculus-ingest-cronjob-{{ $key }}
+          annotations:
+            argocd.argoproj.io/sync-options: Replace=true
+            reloader.stakater.com/auto: "true"
+        spec:
+          restartPolicy: Never
+          containers:
+            - name: ingest-{{ $key }}
+              image: {{ $value.ingest.image}}:{{ $dockerTag }}
+              imagePullPolicy: Always
+              resources:
+                requests:
+                  memory: "1Gi"
+                  cpu: "200m"
+                limits:
+                  cpu: "200m"
+                  memory: "10Gi"
+              env:
+                - name: KEYCLOAK_INGEST_PASSWORD
+                  valueFrom:
+                    secretKeyRef:
+                      name: service-accounts
+                      key: insdcIngestUserPassword
+              args:
+                - snakemake
+                - results/submitted
+                - results/revised
+                - results/approved
+                - --all-temp # Reduce disk usage by not keeping files around
+          {{- if $value.ingest.configFile }}
+              volumeMounts:
+                - name: loculus-ingest-config-volume-{{ $key }}
+                  mountPath: /package/config/config.yaml
+                  subPath: config.yaml
+          volumes:
             - name: loculus-ingest-config-volume-{{ $key }}
-              mountPath: /package/config/config.yaml
-              subPath: config.yaml
-      volumes:
-        - name: loculus-ingest-config-volume-{{ $key }}
-          configMap:
-            name: loculus-ingest-config-{{ $key }}
-      {{- end }}
+              configMap:
+                name: loculus-ingest-config-{{ $key }}
+          {{- end }}
 ---
 apiVersion: batch/v1
 kind: CronJob
 metadata:
-  name: loculus-ingest-cronjob-{{ $key }}
+  name: loculus-revoke-and-regroup-cronjob-{{ $key }}
 spec:
-  schedule: "*/30 * * * *" # ingest every 30 minutes (not more often to be kind to NCBI)
+  schedule: "0 0 31 2 *" # Never runs without manual trigger
   startingDeadlineSeconds: 60
   concurrencyPolicy: Forbid
   jobTemplate:
@@ -107,6 +100,8 @@ spec:
                 - snakemake
                 - results/submitted
                 - results/revised
+                - results/revoked
+                - results/approved
                 - --all-temp # Reduce disk usage by not keeping files around
           {{- if $value.ingest.configFile }}
               volumeMounts: