Develop

.github/workflows/image.yml

@@ -6,7 +6,7 @@ name: CI
 on:
   # Triggers the workflow on push or pull request events but only for the "main" branch
   push:
-    branches: [ "main", "develop" ]
+    branches: [ "main", "develop", "5-dataverse-dag" ]
   pull_request:
     branches: [ "main" ]
 
@@ -42,9 +42,9 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Build the Docker image
-        run: docker build ./accelerator/accel-worker-image -t ghcr.io/niehs/accel-airflow:latest
+        run: docker build ./accelerator/accel-worker-image -t ghcr.io/niehs/accel-airflow:${{ github.ref_name }}
 
       - name: publish
-        run:  docker push ghcr.io/niehs/accel-airflow:latest
+        run:  docker push ghcr.io/niehs/accel-airflow:${{ github.ref_name }}
 
 

README.md

@@ -7,6 +7,7 @@ Helm charts and support for Navigator/Accelerator infrastructure
 ## Links:
 
 * Helm Docs - https://helm.sh/docs/
+* Airflow Command Ref - https://airflow.apache.org/docs/apache-airflow/stable/cli-and-env-variables-ref.html
 
 
 ## Basic usage
@@ -32,8 +33,27 @@ Here I cd into the accelerator subdir of the repo, give a namespace and point to
 ```sh
 kubectl create namespace accelerator-dev
 
-helm repo add bitnami https://charts.bitnami.com/bitnami
-helm repo add airflow https://airflow.apache.org/
+```
+
+Now set up secrets...
+
+Postgres admin password
+
+```sh
+
+# Admin/superuser credentials
+kubectl create secret generic postgres-credentials --from-literal=postgres-password=your-admin-password
+
+
+```
+
+This secret is expected in the default values.yaml.
+
+
+
+```sh
+
+cd accelerator
 
 helm dependency build 
 
@@ -45,7 +65,7 @@ helm install -f ../../accel-values/accel-values.yaml -n accelerator-dev accelera
 here i have alias k="kubectl" and am setting the default namespace as above
 
 ```sh
-k config set-context --current --namespace=accelerator-dev
+k  config set-context --current --namespace accelerator
 
 ```
 
@@ -58,9 +78,156 @@ Here we're uninstalling
 helm uninstall accelerator -ns accelerator-dev
 ```
 
+
+### Staging
+
+For staging, the procedures are the same, with differing namespaces
+
+
+
+```sh
+
+k config set-context --current --namespace=ods-test
+
+helm repo add bitnami https://charts.bitnami.com/bitnami
+helm repo add airflow https://airflow.apache.org/
+
+cd accelerator
+
+helm dependency build 
+
+helm install -f ../../accel-values/accel-values-staging.yaml -n ods-test accelerator .
+
+
+```
+
+
 ## Custom Accelerator Worker
 
 This repo contains a custom airflow container for workers that includes some additional system and python dependencies.
 See the [accel-worker-image](./accelerator/accel-worker-image/README.md)
 
 The build of the custom container is automated by a GitHub action and the image is stored at: ghcr.io/niehs/accel-airflow:TAG, with latest being the default
+
+## Accelerator Variables
+
+For development environments, an example Accelerator variable set looks like this:
+
+```json
+
+{
+    "accelerator.xcom.tempfiles.supported": true,
+    "accelerator.xcom.tempfile.path": "/opt/xcom"
+}
+
+
+```
+
+## Connections
+
+A connection to Mongo needs to be created as follows
+
+![image](https://github.com/user-attachments/assets/c81343c1-2f2a-4f3c-ad53-eaef30ba169d)
+
+Suggested values for local:
+
+
+connection id: mongo_default
+connection type: http
+description: rs0 
+host: mongo-service
+schema: admin
+login: root
+password: ---password as set in helm values.yaml---
+port:  
+
+
+# Notes
+
+
+## Connections page shows error
+
+If you run into an error where you cannot see connections in Airflow, it may be due to the fernet key, you can follow these steps:
+
+1) Shell into the web container
+2) Shell into the database running ```sh airflow db shell ```
+3) Delete the connections from the db: ```sh delete from connection; ```
+
+
+you should now be able to go in and edit connections
+
+
+## redis pod does not terminate
+
+Forcefully delete it with:
+
+```
+k delete pod --grace-period=0 --force accelerator-redis-0
+```
+
+NB this uses the k alias for kubectl
+
+## importing and exporting connections
+
+See https://airflow.apache.org/docs/apache-airflow/stable/howto/usage-cli.html#cli-export-connections
+
+## staging
+
+Use 
+
+```
+helm install -f ../../accel-values/accel-values-staging.yaml -n ods-test accelerator .
+```
+
+## connection string tips
+```
+from pymongo import MongoClient
+
+client = MongoClient(
+    "mongodb://<username>:<password>@"
+    "mongo-service-0.mongo-service-headless.accelerator-dev.svc.cluster.local:27017/"
+    "?replicaSet=rs0&authSource=admin"
+)
+
+db = client["<your_database_name>"]
+```
+### If you don’t care about replica set behavior locally
+
+You can simplify:
+
+mongodb://root:PASSWORD@localhost:27017/admin
+
+
+(no replicaSet, no directConnection)
+
+###add In-cluster apps (no port-forward, running in Kubernetes)
+
+Those should not use localhost. They should use the service DNS name and can omit directConnection=true, e.g.:
+
+mongodb://root:PASSWORD@mongo-service.accelerator-dev.svc.cluster.local:27017/admin?replicaSet=rs0
+
+
+(Or just mongo-service:27017 from same namespace.)
+
+An example configuration as an airflow connection for mongo:
+
+name: mongo_default
+type: http
+
+description: rs0
+host: x
+user: root
+password: xxxx
+
+port: 27017
+schema: accelerator
+
+# tunneling to Mongo
+
+port forward mongodb pod on server using k9s
+
+open ssh tunnel on local machine: ➜  ~ ssh -L 27017:localhost:27017 user@accelserver
+
+update mongodb compass connection string to: mongodb://root:xxxxxxxx@127.0.0.1:27017/?directConnection=true&authSource=admin
+
+

accelerator/Chart.lock

@@ -1,9 +1,12 @@
 dependencies:
 - name: mongodb
   repository: https://charts.bitnami.com/bitnami
-  version: 16.5.8
+  version: 18.1.23
+- name: postgres
+  repository: https://cloudpirates-io.github.io/helm-charts
+  version: 0.12.4
 - name: airflow
   repository: https://airflow.apache.org/
-  version: 1.16.0
-digest: sha256:13b23acf7604dd97db8fac3496ca48a0c88ca21c6d09d36d4d16eaac3245af41
-generated: "2025-05-16T13:21:37.691183-04:00"
+  version: 1.18.0
+digest: sha256:b46f4454e59d72342fffef9be7313478f1988c5fd65c81dd4594aff9002d4ef4
+generated: "2026-01-29T14:29:29.405314-05:00"

accelerator/Chart.yaml

@@ -1,31 +1,30 @@
 apiVersion: v2
 name: accelerator
 description: A Helm chart for deployment of the Accelerator infrastructure
-
 type: application
+
+version: "0.1.3"        # ⬅️ REQUIRED: chart version (semver)
+appVersion: "1.16.3"    # optional: your app version
+
 sources:
   - https://github.com/NIEHS/accelerator-helm
   - https://github.com/apache/airflow
   - https://github.com/bitnami/charts/tree/main/bitnami/postgresql
-  - https://github.com/bitnami/charts/tree/main/bitnami/mongodb
-
-# This is the chart version. This version number should be incremented each time you make changes
-# to the chart and its templates, including the app version.
-# Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.1
-
-# This is the version number of the application being deployed. This version number should be
-# incremented each time you make changes to the application. Versions are not expected to
-# follow Semantic Versioning. They should reflect the version the application is using.
-# It is recommended to use it with quotes.
-appVersion: "1.16.0"
+  - https://github.com/CloudPirates-io/helm-charts/tree/main/charts/mongodb
 
 dependencies:
-- name: mongodb
-  version: ~16.5.1
-  repository: "https://charts.bitnami.com/bitnami"
-- name: airflow 
-  version:  ~1.16.0
-  repository: "https://airflow.apache.org/"
+  #- name: mongodb
+  #  version: "~0.6.6"
+  #  repository: "https://cloudpirates-io.github.io/helm-charts"
+
+  - name: mongodb
+    version: ~18.1.10
+    repository: "https://charts.bitnami.com/bitnami"
 
+  - name: postgres 
+    version: "~0.12.4"
+    repository: "https://cloudpirates-io.github.io/helm-charts"
 
+  - name: airflow 
+    version: "~1.18.0"
+    repository: "https://airflow.apache.org/"

accelerator/accel-worker-image/Dockerfile

@@ -1,8 +1,13 @@
-FROM apache/airflow:2.10.5
+FROM apache/airflow:3.0.2
 USER root
 COPY requirements.txt /
+#COPY cacert.pem /home/airflow/.local/lib/python3.12/site-packages/certifi/
+COPY NIH-FULL.crt /usr/local/share/ca-certificates/
+COPY toxpipe.crt /etc/ssl/certs/
+ENV  REQUESTS_CA_BUNDLE=/home/airflow/.local/lib/python3.12/site-packages/certifi/cacert.pem
 
-RUN apt-get update && apt-get install -y git
+RUN apt-get update && apt-get install -y git nano wget less
+RUN update-ca-certificates
 USER airflow
 RUN pip install --no-cache-dir -r /requirements.txt
 

accelerator/accel-worker-image/NIH-FULL.crt

@@ -0,0 +1,75 @@
+-----BEGIN CERTIFICATE-----
+MIIF3zCCA8egAwIBAgIQPPMxRkhpgrdLNl+H1sUiuzANBgkqhkiG9w0BAQsFADA2
+MQswCQYDVQQGEwJVUzEMMAoGA1UEChMDTklIMRkwFwYDVQQDExBOSUgtRFBLSS1S
+T09ULTFBMB4XDTE1MDkxNzE2MzgzOFoXDTQ1MDkxNzE2MzgzOFowNjELMAkGA1UE
+BhMCVVMxDDAKBgNVBAoTA05JSDEZMBcGA1UEAxMQTklILURQS0ktUk9PVC0xQTCC
+AiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBANBbrArTeMaGgiP1m+z6ahqz
+Gtd6Gv7R7mNKU4h+6J2tz091Vn+dBTzKouzBO9fmL2j8sw5XdZSjiy5cSEla1yGC
+yHeTa/YnqTZ3KIwZgZ9S3fRvtNZpZw20a49coA97rfZ+YSSq7l1jJKdMNLxnIBe6
+kjtWJs+veC0C6f+2R5ARJWIIFVUFMGzjvKvrYLda16F1suKkAEQRQTCeFvZ3RKvw
+CJMcHQPslqd+jXw3Fqfkg2wqd5XChmZ2hoKGwKcwI07R/m7kvdxZCD1bmm7C1hee
+7mWZPZ0UuKa93laF8kym4h34ooF+Ijr0D3ap36GpqiShB7ObNNxc/bRVNkYJwk07
+ENzItwXjDkCggs73jtjKZfMwUZL+jwLH+pJRFRJ9oHVihsDSnk1r8T6bg7L2u/F+
+a5Q43hQ39HtkwtpAeNAMawduh9J9JCz4lD1EF9nPVIU4yihwY8I4ykcJK7mFIpEh
+I2Q++KiAgug9IMKkaQwNej+wKqzu7RBgPqdrl3KH9Eu2tHrLrjrO0AsbwcA+dN9J
+kMRjgjz/uk4z2ndGI6v7CzLzBuvdVUJOCTFbQMeITUp/6RLx1DeLdw+djhhaZ6py
+9k8dzlQE01PBB0Wk3GuaEw3ubxtE6+H5BsaIG2ZQ0WZVff58a4FwKmiSzxqqblrX
+Keb/C18c91tkeQ/LLW0LAgMBAAGjgegwgeUwCwYDVR0PBAQDAgGGMBIGA1UdEwEB
+/wQIMAYBAf8CAQEwHQYDVR0OBBYEFFI4U+ocW7rBl3bDbR1JgbsNOvSkMBAGCSsG
+AQQBgjcVAQQDAgEAMIGQBgNVHSAEgYgwgYUwgYIGDGCGSAFlAwIBDQUBAjByMEgG
+CCsGAQUFBwICMDweOgBOAEkASAAgAEMAUABTACAAUABvAGwAaQBjAHkAIABTAHQA
+YQB0AGUAbQBlAG4AdAAgAFQAZQB4AHQwJgYIKwYBBQUHAgEWGmh0dHA6Ly9zbWFy
+dGNhcmQubmloLmdvdi8AMA0GCSqGSIb3DQEBCwUAA4ICAQDBIZpPIiUDd40qcYVK
+fnEFSoot0r/8XTIax9ZpAFGIu+NheMbUY6wNhijPG5lQdYx+BgloCIv776+AShli
+CJbQ+GBs07lyhY0b4jJ4VEv778WQPZ7IDQ+XIpnC7DLEqB2iRQS/ohGIsNqOKaw5
+O4yedH/K1BRlMOR18sFru220BlJj5g8RzxCJoLR5zaaK9IgxKTMrbMveQ3V2AJlx
+HbqBu6f+sGG9kOlCAFdPplX0VJ13p1DnSnHFPeO4jDRDcGB7JchZPnQxPa/qhVhs
+eexHju6cl7bY1CrtjHIYrIFLNQLVFCMmcZXMO0JZcRNqV2baZ/RmrJLz3jMc6KO1
+el9zD+RzJqC973PTRiYgCwSzyXRD+T3L2c6mLzpYwVbxni6Ay5RcT+MQwpdVqBsI
+qGde3e16bCDJjKV2p6DeY7spblGXA2eWfQSVp097131F/kpbZ7e8f4RrKuzDHLyz
+ZlBfgZvbonuL05NZyCpm32TgTax3GhOeeQSyPwgniy6Ob67f3mzmwULZXlzxH99y
+Zspp8NPLcG//A0AKT77kVJwkcTSqRKSHbjYpMv0HGdtTKZSBKM7fxcrfL1XQoOsT
+PzYKxZZ6OlI2g1IyQunzsGGFoLCSowHf1618Mr1SufFoutl6h4aQxtvCNzrVlM7p
+H7TZPbA+zMPpoy3VxsczwrwjWg==
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIHHzCCBQegAwIBAgITJgAAABPun/MhT8G65wAAAAAAEzANBgkqhkiG9w0BAQsF
+ADA2MQswCQYDVQQGEwJVUzEMMAoGA1UEChMDTklIMRkwFwYDVQQDExBOSUgtRFBL
+SS1ST09ULTFBMB4XDTIzMDYxNDE0NTkzOVoXDTMzMDYxNDE1MDkzOVowTjEVMBMG
+CgmSJomT8ixkARkWBUxPQ0FMMRwwGgYKCZImiZPyLGQBGRYMREhIU1NFQ1VSSVRZ
+MRcwFQYDVQQDEw5OSUgtRFBLSS1DQS0xQTCCASIwDQYJKoZIhvcNAQEBBQADggEP
+ADCCAQoCggEBANbUoi2rFCfOI/EKeY4A4qYBVJGC6snhYxR3YdeLEV8rREWyWMGk
+FXEdDQDtbxTp2Su356PAz9VqcvEaDRLcW7ukdyGOTtqlwMxf5XR70FY2w8lf1vfr
+CTd8U+5FEJrZg2jWJkjN0oazLzQUE+H7MC6ybSMlLRlyB4OzZAaIWjshFEHiHBMl
+f0MmaJkj64lIteO6NDXfiR9W1hCCds+QMjTh436suBgRHKA9hip1uHg2kg2JXM1h
+SbsGw/yuK8lPhnGntZUA5Of3ZfSsGwDIEMf93Ze+vK/n8PtOXFqT2LFUyHQombMv
+FimqyiPg0jzj6jbu5dVeim+ecq+thlBhYEkCAwEAAaOCAwwwggMIMBIGCSsGAQQB
+gjcVAQQFAgMBAAEwIwYJKwYBBAGCNxUCBBYEFGV1GZ5K+P5s2b4PF6h/h4cCZU2I
+MB0GA1UdDgQWBBSagAU2EMkzy98Cspq6dbDyULH00TAZBgNVHSAEEjAQMA4GDGCG
+SAFlAwIBDQUBAjAZBgkrBgEEAYI3FAIEDB4KAFMAdQBiAEMAQTALBgNVHQ8EBAMC
+AYYwEgYDVR0TAQH/BAgwBgEB/wIBADAfBgNVHSMEGDAWgBRSOFPqHFu6wZd2w20d
+SYG7DTr0pDCCARUGA1UdHwSCAQwwggEIMIIBBKCCAQCggf2GgcFsZGFwOi8vL0NO
+PU5JSC1EUEtJLVJPT1QtMUEsQ049TklIRFBLSVJDQSxDTj1DRFAsQ049UHVibGlj
+JTIwS2V5JTIwU2VydmljZXMsQ049U2VydmljZXMsQ049Q29uZmlndXJhdGlvbixE
+Qz1ESEhTU2VjdXJpdHksREM9bG9jYWw/Y2VydGlmaWNhdGVSZXZvY2F0aW9uTGlz
+dD9iYXNlP29iamVjdENsYXNzPWNSTERpc3RyaWJ1dGlvblBvaW50hjdodHRwOi8v
+TklIRFBLSUNSTC5OSUguR09WL0NlcnREYXRhL05JSC1EUEtJLVJPT1QtMUEuY3Js
+MIIBGwYIKwYBBQUHAQEEggENMIIBCTCBtgYIKwYBBQUHMAKGgalsZGFwOi8vL0NO
+PU5JSC1EUEtJLVJPT1QtMUEsQ049QUlBLENOPVB1YmxpYyUyMEtleSUyMFNlcnZp
+Y2VzLENOPVNlcnZpY2VzLENOPUNvbmZpZ3VyYXRpb24sREM9REhIU1NlY3VyaXR5
+LERDPWxvY2FsP2NBQ2VydGlmaWNhdGU/YmFzZT9vYmplY3RDbGFzcz1jZXJ0aWZp
+Y2F0aW9uQXV0aG9yaXR5ME4GCCsGAQUFBzAChkJodHRwOi8vTklIRFBLSUNSTC5O
+SUguR09WL0NlcnREYXRhL05JSERQS0lSQ0FfTklILURQS0ktUk9PVC0xQS5jcnQw
+DQYJKoZIhvcNAQELBQADggIBACrWNohYPCSQ1JZ964ym9s+fm4QXv2NQbzAC9l/m
+zQ8PXH3HU2zMO9exVvWQOLJgxjxyahX4T6Hjznx/5hReO5Hw07XRG+CvEahARu9p
+q5FiI/pWoZlaSJsTbH81Kwov1Zvcg3FrYzrKfSpa851yL434nwfMQ4EtOzFKXDNI
+Hvzf177hKF+ruFujHLDzRIJ2e1fbIj+XRa1iqLF2ryknQeTtrd9Blst08A74kkNM
+jFyHbRr0idNJFAzK2CDHprr+82p8SYvJ6R6tn6Q7fpWd3ZeAwuhs082iUSU+CqNn
+dQhDJAxSfuB2kkL+ULo14ZAeyEJWmjQCopjzwxZykJzYoQP8YW3lSw4KeMw3g1H2
+OiPza//Cz+LpQ0BamTa7H6NxQ53PtiCcBsODIFLNioJ5SZVoHAIjSgCSpfSfGGMF
+WNS+uZJUv+uOgGMAP/eryn5h7suPZ1HgilkKQWJ79K83Beckpl51Klh11sdeBYfI
+4Avho3HffJH2F0twovQD31lkufjVtPX31VdffTSg7jnkJfDf4q0OiwX/CDxi4eKu
+cJ9r+eoLXErKZZjdzvokjzv0PRCSZQX+9qDWEByZSUBIcGWrdurp2yL7Ll8EFhDS
+BZUo2yKD7QA+LLc9IiWo+aGg7iUtzKQpgLa9FjZZcgJ2Wb/BDboxQlFohmgbGd7J
+b3Ap
+-----END CERTIFICATE-----