feat (keycloak): add email verification, password reset to keycloak (#…

…1320) Adds email functionality to keycloak. To get the password to keycloak, the config processor is updated to additionally substitute from environmental variables (which can contain secrets). Also we rename "loculusRealm" to "loculus" - in part as the emails surface this name. We are currently using Theo's Mailjet account - we probably can for a while - it has a limit of 15k emails per month, split with some other projects.
loculus-project · Mar 11, 2024 · 04bce57 · 04bce57
1 parent c62b4b5
commit 04bce57
Show file tree

Hide file tree

Showing 17 changed files with 106 additions and 26 deletions.
diff --git a/.idea/runConfigurations/BackendApplicationKt.run.xml b/.idea/runConfigurations/BackendApplicationKt.run.xml
diff --git a/README.md b/README.md
@@ -37,11 +37,11 @@ Follow this guide <https://docs.github.com/en/packages/working-with-a-github-pac
 
 ### User management
 
-We use keycloak for authorization. The keycloak instance is deployed in the `loculus` namespace and exposed to the outside either under `localhost:8083` or `authentication-[your-argo-cd-path]`. The keycloak instance is configured with a realm called `loculusRealm` and a client called `test-cli`. The realm is configured to use the exposed url of keycloak as a [frontend url](https://www.keycloak.org/server/hostname).
+We use keycloak for authorization. The keycloak instance is deployed in the `loculus` namespace and exposed to the outside either under `localhost:8083` or `authentication-[your-argo-cd-path]`. The keycloak instance is configured with a realm called `loculus` and a client called `test-cli`. The realm is configured to use the exposed url of keycloak as a [frontend url](https://www.keycloak.org/server/hostname).
 For testing we added multiple users to the realm. The users are:
 
 - `admin` with password `admin` (login under `your-exposed-keycloak-url/admin/master/console/`)
-- `testuser` with password `testuser` (login under `your-exposed-keycloak-url/realms/loculusRealm/account/`)
+- `testuser` with password `testuser` (login under `your-exposed-keycloak-url/realms/loculus/account/`)
 - and more testusers, for each browser in the e2e test following the pattern: `testuser_[processId]_[browser]` with password `testuser_[processId]_[browser]`
 - These testusers will be added to the `testGroup` in the setup for e2e tests. If you change the number of browsers in the e2e test, you need to adapt `website/tests/playwrightSetup.ts` accordingly.
 - To validate that a user exists we also created a technical user for the backend with username `backend` and password `backend`. The technical user is authorized to view users and groups and in principle to manage its own account.

diff --git a/backend/README.md b/backend/README.md
@@ -37,7 +37,7 @@ You need to set:
 * the url to fetch the public key for JWT verification 
   (corresponding to the `jwks_uri` value in the `/.well-known/openid-configuration` endpoint of the Keycloak server):
 ```
---spring.security.oauth2.resourceserver.jwt.jwk-set-uri=http://localhost:8083/realms/loculusRealm/protocol/openid-connect/certs
+--spring.security.oauth2.resourceserver.jwt.jwk-set-uri=http://localhost:8083/realms/loculus/protocol/openid-connect/certs
 ```
 
 We use Flyway, so that the service can provision an empty/existing DB without any manual steps in between. On startup scripts in `src/main/resources/db/migration` are executed in order, i.e. `V1__*.sql` before `V2__*.sql` if they didn't run before, so that the DB is always up-to-date. (For more info on the naming convention, see [this](https://www.red-gate.com/blog/database-devops/flyway-naming-patterns-matter) blog post.)

diff --git a/backend/start_dev.sh b/backend/start_dev.sh
@@ -1,2 +1,2 @@
 #! /bin/sh
-./gradlew bootRun --args="--spring.datasource.url=jdbc:postgresql://localhost:5432/loculus --spring.datasource.username=postgres --spring.datasource.password=unsecure --loculus.config.path=../website/tests/config/backend_config.json --spring.security.oauth2.resourceserver.jwt.jwk-set-uri=http://localhost:8083/realms/loculusRealm/protocol/openid-connect/certs --keycloak.user=backend --keycloak.password=backend --keycloak.realm=loculusRealm --keycloak.client=test-cli --keycloak.url=http://localhost:8083"
+./gradlew bootRun --args="--spring.datasource.url=jdbc:postgresql://localhost:5432/loculus --spring.datasource.username=postgres --spring.datasource.password=unsecure --loculus.config.path=../website/tests/config/backend_config.json --spring.security.oauth2.resourceserver.jwt.jwk-set-uri=http://localhost:8083/realms/loculus/protocol/openid-connect/certs --keycloak.user=backend --keycloak.password=backend --keycloak.realm=loculus --keycloak.client=test-cli --keycloak.url=http://localhost:8083"
diff --git a/get_testuser_token.sh b/get_testuser_token.sh
@@ -1,7 +1,7 @@
 #!/usr/bin/bash
 set -eu
 
-KEYCLOAK_TOKEN_URL="http://localhost:8083/realms/loculusRealm/protocol/openid-connect/token"
+KEYCLOAK_TOKEN_URL="http://localhost:8083/realms/loculus/protocol/openid-connect/token"
 KEYCLOAK_CLIENT_ID="test-cli"
 
 echo "Retrieving JWT from $KEYCLOAK_TOKEN_URL"

diff --git a/kubernetes/config-processor/config-processor.py b/kubernetes/config-processor/config-processor.py
@@ -3,6 +3,7 @@
 import requests
 import re
 
+
 def copy_structure(input_dir, output_dir):
     for root, dirs, files in os.walk(input_dir):
         for dir in dirs:
@@ -20,27 +21,42 @@ def replace_url_with_content(file_content):
             file_content = file_content.replace(f"[[URL:{url}]]", response.text)
     return file_content
 
-def process_files(output_dir):
+def make_substitutions(file_content, substitutions):
+    for key, value in substitutions.items():
+        file_content = file_content.replace(f"[[{key}]]", value)
+    return file_content
+
+def process_files(output_dir, substitutions):
     for root, dirs, files in os.walk(output_dir):
         for file in files:
             file_path = os.path.join(root, file)
             with open(file_path, 'r+') as f:
                 print(f"Processing {file_path}")
                 content = f.read()
                 new_content = replace_url_with_content(content)
+                new_content = make_substitutions(new_content, substitutions)
                 if new_content != content:
                     f.seek(0)
                     f.write(new_content)
                     f.truncate()
 
-def main(input_dir, output_dir):
+def main(input_dir, output_dir, substitutions):
     print(f"Processing {input_dir} to {output_dir}")
     copy_structure(input_dir, output_dir)
     print(f"Copied directory structure from {input_dir} to {output_dir}")
-    process_files(output_dir)
+    process_files(output_dir, substitutions)
 
 if __name__ == "__main__":
     import sys
     input_dir = sys.argv[1]
     output_dir = sys.argv[2]
-    main(input_dir, output_dir)
+
+
+    substitutions = {}
+    for var in os.environ:
+        sub_start = "LOCULUSSUB_"
+        if var.startswith(sub_start):
+            key = var[len(sub_start):]
+            value = os.environ[var]
+            substitutions[key] = value
+    main(input_dir, output_dir, substitutions)
diff --git a/kubernetes/loculus/templates/_config-processor.tpl b/kubernetes/loculus/templates/_config-processor.tpl
@@ -9,7 +9,14 @@
       mountPath: /output
   command: ["python3"]
   args: ["/app/config-processor.py", "/input", "/output"]
-{{- end -}}
+  env:
+    - name: LOCULUSSUB_smtpPassword
+      valueFrom:
+        secretKeyRef:
+          name: smtp-password
+          key: secretKey
+{{- end }}
+
 
 {{- define "loculus.configVolume" -}}
 - name: {{ .name }}

diff --git a/kubernetes/loculus/templates/ghcr-secret.yaml b/kubernetes/loculus/templates/ghcr-secret.yaml
@@ -6,6 +6,15 @@ kind: Secret
 metadata:
   name: ghcr-secret
 type: kubernetes.io/dockerconfigjson
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: smtp-password
+type: Opaque
+data:
+  secretKey: {{ "NO_PASSWORD_SET" | b64enc }}
+
 
 {{- else }}
 
@@ -26,6 +35,24 @@ spec:
       name: ghcr-secret
     type: kubernetes.io/dockerconfigjson
 
+---
+apiVersion: bitnami.com/v1alpha1
+kind: SealedSecret
+metadata:
+  annotations:
+    sealedsecrets.bitnami.com/cluster-wide: "true"
+  name: smtp-password
+spec:
+  encryptedData:
+    secretKey: AgCoDKVhu9eUF90E2CXJmF4CVFDHe8zHyQqQLNDaJ4dlCkm2QHvEEBL8WXABAMlfJqIv/W/rHmz2QfhpZRNYGr7r+Q4wk5nvi5bh+HAmAecglsI6UMqTj+hNRVDrgwZ1mevzJKdx2dDQ3kOk6hP8s/EmMsabmPEavCVse/3dE88VNop9HAPlbvuTk2hbFZhavDOmarVxv2vSuPSGOjEch2YwuHai+3LK2F4gVRk5ziDzkc7WhjGlczAW/IKqPo/Y7ecKeopJN+nZp4lMzNHzaiYFX8eegzRnXW0aB1Cop91mLx2PKUGiM4jLmUb/m2tWaLvUPmmrNeblI6yxHoHbu1McPJlQAepUekeoeypxuTxgCDkdTG3CPhpPQjg3XUpdrtjmjuxdljHEhIbmP/8sX+v9I5Px88msQkcUHEkXibjsWmSNChRCN40e61B4VScvEB84SemrEAFyApMbbru2H0G2kKwPMPBqN2f71rDUdqiryh7yMCwZANxBCt6XNxwo7ndss0NNBcE+qLfthHX0mFr9Sn8p09tC8Qo8pg9lKhseT2o/1VEdvXQONi2zy/lo/KNijhrn+Kbhc2Ea3bcFosAsSGxVgbZ1penGmjrYW7XVgRxeQ171oc/Xiy7SBc8wSqfBOamnYzIQkpHfpBnE5KoDzFdkUsQPwA+32FfsZt9PgXiGyVf9dSfO+yQJXFO+dhpWQMk4UfG0wEUB2TyzpI+/HhVhRAz6Bpwdj5YJc+IPTw==
+  template:
+    metadata:
+      annotations:
+        sealedsecrets.bitnami.com/cluster-wide: "true"
+      name: smtp-password
+    type: Opaque
+{{- end }}
+
 {{ if .Values.customWebsiteSealedSecret }}
 ---
 apiVersion: bitnami.com/v1alpha1
@@ -47,4 +74,3 @@ spec:
     type: kubernetes.io/dockerconfigjson
 {{- end }}
 
-{{- end }}
diff --git a/kubernetes/loculus/templates/keycloak-config-map.yaml b/kubernetes/loculus/templates/keycloak-config-map.yaml
@@ -7,8 +7,24 @@ metadata:
 data:
   keycloak-config.json: |
     {
-      "realm": "loculusRealm",
+      "realm": "loculus",
       "enabled": true,
+      "verifyEmail": {{$.Values.auth.verifyEmail}},
+      "resetPasswordAllowed": {{$.Values.auth.resetPasswordAllowed}},
+      "smtpServer": {
+        "host": "{{$.Values.auth.smtp.host}}",
+        "port": "{{$.Values.auth.smtp.port}}",
+        "from": "{{$.Values.auth.smtp.from}}",
+        "fromDisplayName": "{{$.Values.name}}",
+        "replyTo": "{{$.Values.auth.smtp.replyTo}}",
+        "replyToDisplayName": "{{$.Values.name}}",
+        "envelopeFrom": "{{$.Values.auth.smtp.envelopeFrom}}",
+        "ssl": "false",
+        "starttls": "true",
+        "auth": "true",
+        "user": "{{$.Values.auth.smtp.user}}",
+        "password": "[[smtpPassword]]"
+      },
       "registrationAllowed": true,
       "accessTokenLifespan": 36000,
       "ssoSessionIdleTimeout": 36000,
@@ -20,6 +36,7 @@ data:
           "username": "testuser_{{$index}}_{{$browser}}",
           "enabled": true,
           "email": "testuser_{{$index}}_{{$browser}}@keycloak.org",
+          "emailVerified": true,
           "firstName": "Test",
           "lastName": "User",
           "credentials": [
@@ -47,6 +64,7 @@ data:
           "username": "testuser",
           "enabled": true,
           "email": "testuser@keycloak.org",
+          "emailVerified" : true,
           "firstName": "Test",
           "lastName": "User",
           "credentials": [
@@ -72,6 +90,7 @@ data:
           "username": "insdc_ingest_user",
           "enabled": true,
           "email": "insdc_ingest_user@keycloak.org",
+          "emailVerified" : true,
           "firstName": "INSDC Ingest",
           "lastName": "User",
           "credentials": [
@@ -97,6 +116,7 @@ data:
           "username": "dummy_preprocessing_pipeline",
           "enabled": true,
           "email": "dummy_preprocessing_pipeline@keycloak.org",
+          "emailVerified" : true,
           "firstName": "Dummy",
           "lastName": "Preprocessing",
           "credentials": [
@@ -122,6 +142,7 @@ data:
           "username": "silo_import_job",
           "enabled": true,
           "email": "silo_import_job@keycloak.org",
+          "emailVerified": true,
           "firstName": "SILO",
           "lastName": "ImportJob",
           "credentials": [
@@ -147,6 +168,7 @@ data:
           "username": "backend",
           "enabled": true,
           "email": "nothing@void.o",
+          "emailVerified": true,
           "firstName": "Backend",
           "lastName": "Technical-User",
           "attributes": {
@@ -205,12 +227,12 @@ data:
           "description" : "",
           "rootUrl" : "${authBaseUrl}",
           "adminUrl" : "",
-          "baseUrl" : "/realms/loculusRealm/account/",
+          "baseUrl" : "/realms/loculus/account/",
           "surrogateAuthRequired" : false,
           "enabled" : true,
           "alwaysDisplayInConsole" : false,
           "clientAuthenticatorType" : "client-secret",
-          "redirectUris" : [ "/realms/loculusRealm/account/*" ],
+          "redirectUris" : [ "/realms/loculus/account/*" ],
           "webOrigins" : [ "+" ],
           "notBefore" : 0,
           "bearerOnly" : false,

diff --git a/kubernetes/loculus/templates/keycloak-deployment.yaml b/kubernetes/loculus/templates/keycloak-deployment.yaml
@@ -30,6 +30,7 @@ spec:
         component: keycloak
     spec:
       initContainers:
+{{- include "loculus.configProcessor" (dict "name" "keycloak-config" "dockerTag" $dockerTag) | nindent 8 }}
         - name: keycloak-theme-prep
           image: "ghcr.io/loculus-project/keycloakify:{{ $dockerTag }}"
           volumeMounts:
@@ -73,7 +74,7 @@ spec:
           ports:
             - containerPort: 8080
           volumeMounts:
-            - name: config-volume
+            - name: keycloak-config-processed
               mountPath: /opt/keycloak/data/import/
             - name: theme-volume
               mountPath: /opt/keycloak/providers/
@@ -90,9 +91,7 @@ spec:
             initialDelaySeconds: 60
             periodSeconds: 10
       volumes:
-        - name: config-volume
-          configMap:
-            name: keycloak-config
+{{ include "loculus.configVolume" (dict "name" "keycloak-config") | nindent 8 }}
         - name: theme-volume
           emptyDir: {}
       imagePullSecrets:

diff --git a/kubernetes/loculus/templates/lapis-silo-import-cronjob.yaml b/kubernetes/loculus/templates/lapis-silo-import-cronjob.yaml
@@ -1,5 +1,5 @@
 {{- $dockerTag := include "loculus.dockerTag" .Values }}
-{{- $keycloakTokenUrl := "http://loculus-keycloak-service:8083/realms/loculusRealm/protocol/openid-connect/token" }}
+{{- $keycloakTokenUrl := "http://loculus-keycloak-service:8083/realms/loculus/protocol/openid-connect/token" }}
 
 
 {{- range $key, $_ := (.Values.organisms | default .Values.defaultOrganisms) }}

diff --git a/kubernetes/loculus/templates/loculus-deployment.yaml b/kubernetes/loculus/templates/loculus-deployment.yaml
@@ -74,10 +74,10 @@ spec:
             - "--spring.datasource.password=unsecure"
             - "--keycloak.user=backend"
             - "--keycloak.password=backend"
-            - "--keycloak.realm=loculusRealm"
+            - "--keycloak.realm=loculus"
             - "--keycloak.client=test-cli"
             - "--keycloak.url=http://loculus-keycloak-service:8083"
-            - "--spring.security.oauth2.resourceserver.jwt.jwk-set-uri=http://loculus-keycloak-service:8083/realms/loculusRealm/protocol/openid-connect/certs"
+            - "--spring.security.oauth2.resourceserver.jwt.jwk-set-uri=http://loculus-keycloak-service:8083/realms/loculus/protocol/openid-connect/certs"
           volumeMounts:
             - name: loculus-backend-config-processed
               mountPath: /config

diff --git a/kubernetes/loculus/values.yaml b/kubernetes/loculus/values.yaml
@@ -123,3 +123,13 @@ defaultOrganisms:
       genes:
         - name: OPG001
           sequence: MKQYIVLACMCLVAAAMPTSLQQSSSSCTEEENKHHMGIDVIIKVTKQDQTPTNDKICQSVTEVTETEDDEVSEEVVKGDPTTYYTIVGAGLNMNFGFTKCPKISSISESSDGNTVNTRLSSVSPGQGKDSPAITREEALAMIKDCEMSIDIRCSEEEKDSDIKTHPVLGSNISHKKVSYKDIIGSTIVDTKCVKNLEFSVRIGDMCEESSELEVKDGFKYVDGSASEGATDDTSLIDSTKLKACV*
+auth:
+  smtp:
+    host: "in-v3.mailjet.com"
+    port: 587
+    user: "fafd505de339dd2e9c3e85ad9981af8a"
+    replyTo: "noreply@loculus.org"
+    from: "noreply@loculus.org"
+    envelopeFrom: "noreply@loculus.org"
+  verifyEmail: true
+  resetPasswordAllowed: true
diff --git a/preprocessing/dummy/main.py b/preprocessing/dummy/main.py
@@ -21,7 +21,7 @@
                     help="Keycloak user to use for authentication")
 parser.add_argument("--keycloak-password", type=str, default="dummy_preprocessing_pipeline",
                     help="Keycloak password to use for authentication")
-parser.add_argument("--keycloak-token-path", type=str, default="/realms/loculusRealm/protocol/openid-connect/token", help="Path to Keycloak token endpoint")
+parser.add_argument("--keycloak-token-path", type=str, default="/realms/loculus/protocol/openid-connect/token", help="Path to Keycloak token endpoint")
 
 args = parser.parse_args()
 backendHost = args.backend_host

diff --git a/preprocessing/nextclade/src/loculus_preprocessing/config.py b/preprocessing/nextclade/src/loculus_preprocessing/config.py
@@ -21,7 +21,7 @@ class Config:
     keycloak_host: str = "http://172.0.0.1:8083"
     keycloak_user: str = "dummy_preprocessing_pipeline"
     keycloak_password: str = "dummy_preprocessing_pipeline"
-    keycloak_token_path: str = "realms/loculusRealm/protocol/openid-connect/token"
+    keycloak_token_path: str = "realms/loculus/protocol/openid-connect/token"
     nextclade_dataset_name: str = "nextstrain/mpox/all-clades"
     nextclade_dataset_version: str = "2024-01-16--20-31-02Z"
     config_file: str | None = None

diff --git a/website/src/middleware/authMiddleware.ts b/website/src/middleware/authMiddleware.ts
@@ -26,7 +26,7 @@ export const clientMetadata = {
     public: true,
 };
 
-export const realmPath = '/realms/loculusRealm';
+export const realmPath = '/realms/loculus';
 
 let _keycloakClient: BaseClient | undefined;
 

diff --git a/website/tests/pages/submission/index.spec.ts b/website/tests/pages/submission/index.spec.ts
@@ -9,7 +9,7 @@ test.describe('The submit page', () => {
 
         await submitPage.loginButton.click();
 
-        expect(submitPage.page.url()).toContain('loculusRealm');
+        expect(submitPage.page.url()).toContain('realms/loculus');
     });
 
     test('should upload files and submit', async ({ submitPage, loginAsTestUser }) => {