From f3719bb3aefadab4f040d50b58c01e892129abc7 Mon Sep 17 00:00:00 2001 From: Alyssa Dai Date: Wed, 13 Nov 2024 16:01:33 -0500 Subject: [PATCH 1/5] add instructions for using docker-compose.yml with NGINX+ACME --- docs/config.md | 162 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 162 insertions(+) diff --git a/docs/config.md b/docs/config.md index ea367b1..ae26d71 100644 --- a/docs/config.md +++ b/docs/config.md @@ -164,6 +164,168 @@ Ensure that you not accidentally provide the address of your actual federation A To add one or more local nodes to the list of nodes known to your f-API, simply add more dictionaries to this file. +## Behind a reverse proxy + +!!! warning "For advanced users" + +To allow access to your Neurobagel node using a custom domain name, you will likely want to set up a reverse proxy (e.g., [NGINX](https://nginx.org/en/docs/beginners_guide.html), [Caddy](https://caddyserver.com/docs/quick-starts/reverse-proxy)) for your services. +This will route incoming requests from your custom domain(s) to your local Neurobagel node API, your local Neurobagel query tool, etc. + +Below is an example implementation of a reverse proxy, using a custom Docker Compose file that builds on [`docker-compose.yml`](https://github.com/neurobagel/recipes/blob/main/docker-compose.yml) in the default Neurobagel deployment recipe. + +This file adds: + +- [nginx-proxy](https://github.com/nginx-proxy/nginx-proxy) to run NGINX in a container, and automatically generate/refresh NGINX reverse proxy configurations when Neurobagel service containers are started +- [acme-companion](https://github.com/nginx-proxy/acme-companion) to automatically create and renew SSL certificates for NGINX proxied service containers + +??? abstract "Example `docker-compose.yml` with NGINX reverse proxy" + To use this file: + + 1. If you haven't already, follow the [steps](getting_started.md#the-neurobagel-node-deployment-recipe) to clone and minimally configure the services in the [Neurobagel deployment recipe](https://github.com/neurobagel/recipes) + 2. Replace the default `docker-compose.yml` in the `recipes` directory with the below file + 2. Open the file, and edit the values of `VIRTUAL_HOST` and `LETSENCRYPT_HOST` to the domains your proxied services will use (see :material-plus-circle:) + - This assumes you have already registered your domain(s) with a DNS provider and configured the DNS settings to resolve correctly to your host machine + + 3. From the root of the `recipes` directory, run: + ```bash + docker compose up -d + ``` + (or, see [here](#available-profiles) to launch a non-default service profile) + 4. Ensure that ports 80 and 443 are reachable for the host machine where your Docker Compose stack is running + + ```{ .yaml .annotate title="docker-compose.yml" } + services: + + api: + image: "neurobagel/api:${NB_NAPI_TAG:-latest}" + profiles: + - "local_node" + - "full_stack" + ports: + - "${NB_NAPI_PORT_HOST:-8000}:8000" + environment: + NB_GRAPH_USERNAME: ${NB_GRAPH_USERNAME} + NB_GRAPH_ADDRESS: graph + NB_GRAPH_PORT: 7200 + NB_GRAPH_DB: ${NB_GRAPH_DB:-repositories/my_db} + NB_RETURN_AGG: ${NB_RETURN_AGG:-true} + NB_API_PORT: 8000 + NB_API_ALLOWED_ORIGINS: ${NB_NAPI_ALLOWED_ORIGINS} + NB_ENABLE_AUTH: ${NB_ENABLE_AUTH:-false} + NB_QUERY_CLIENT_ID: ${NB_QUERY_CLIENT_ID} + VIRTUAL_HOST: myservice1.myinstitute.org # (1)! + LETSENCRYPT_HOST: myservice1.myinstitute.org # (2)! + VIRTUAL_PORT: 8000 + volumes: + - "./scripts/api_entrypoint.sh:/usr/src/api_entrypoint.sh" + entrypoint: + - "/usr/src/api_entrypoint.sh" + secrets: + - db_user_password + + graph: + image: "ontotext/graphdb:10.3.1" + profiles: + - "local_node" + - "full_stack" + volumes: + - "graphdb_home:/opt/graphdb/home" + - "./scripts:/usr/src/neurobagel/scripts" + - "./vocab:/usr/src/neurobagel/vocab" + - "${LOCAL_GRAPH_DATA:-./data}:/data" + ports: + - "${NB_GRAPH_PORT_HOST:-7200}:7200" + environment: + NB_GRAPH_USERNAME: ${NB_GRAPH_USERNAME} + NB_GRAPH_PORT: 7200 + NB_GRAPH_DB: ${NB_GRAPH_DB:-repositories/my_db} + entrypoint: + - "/usr/src/neurobagel/scripts/setup.sh" + working_dir: "/usr/src/neurobagel/scripts" + secrets: + - db_admin_password + - db_user_password + + federation: + image: "neurobagel/federation_api:${NB_FAPI_TAG:-latest}" + profiles: + - "local_federation" + - "full_stack" + ports: + - "${NB_FAPI_PORT_HOST:-8080}:8000" + volumes: + - "./local_nb_nodes.json:/usr/src/local_nb_nodes.json:ro" + environment: + NB_API_PORT: 8000 + NB_FEDERATE_REMOTE_PUBLIC_NODES: ${NB_FEDERATE_REMOTE_PUBLIC_NODES:-True} + NB_ENABLE_AUTH: ${NB_ENABLE_AUTH:-false} + NB_QUERY_CLIENT_ID: ${NB_QUERY_CLIENT_ID} + VIRTUAL_HOST: myservice2.myinstitute.org # (3)! + LETSENCRYPT_HOST: myservice2.myinstitute.org # (4)! + VIRTUAL_PORT: 8000 + + query_federation: + image: "neurobagel/query_tool:${NB_QUERY_TAG:-latest}" + profiles: + - "local_federation" + - "full_stack" + ports: + - "${NB_QUERY_PORT_HOST:-3000}:5173" + environment: + NB_API_QUERY_URL: ${NB_API_QUERY_URL} + NB_QUERY_APP_BASE_PATH: ${NB_QUERY_APP_BASE_PATH:-/} + NB_ENABLE_AUTH: ${NB_ENABLE_AUTH:-false} + NB_QUERY_CLIENT_ID: ${NB_QUERY_CLIENT_ID} + VIRTUAL_HOST: myservice3.myinstitute.org # (5)! + LETSENCRYPT_HOST: myservice3.myinstitute.org # (6)! + VIRTUAL_PORT: 5173 + + nginx-proxy: + image: nginxproxy/nginx-proxy + container_name: nginx-proxy + ports: + - "80:80" + - "443:443" + volumes: + - conf:/etc/nginx/conf.d + - vhost:/etc/nginx/vhost.d + - html:/usr/share/nginx/html + - certs:/etc/nginx/certs:ro + - /var/run/docker.sock:/tmp/docker.sock:ro + + acme-companion: + image: nginxproxy/acme-companion + container_name: nginx-proxy-acme + volumes_from: + - nginx-proxy + volumes: + - certs:/etc/nginx/certs:rw + - acme:/etc/acme.sh + - /var/run/docker.sock:/var/run/docker.sock:ro + + secrets: + db_admin_password: + file: ${NB_GRAPH_SECRETS_PATH:-./secrets}/NB_GRAPH_ADMIN_PASSWORD.txt + db_user_password: + file: ${NB_GRAPH_SECRETS_PATH:-./secrets}/NB_GRAPH_PASSWORD.txt + + volumes: + graphdb_home: + conf: + vhost: + html: + certs: + acme: + ``` + + 1. Replace with your custom domain + 2. Replace with your custom domain (should be same as `VIRTUAL_HOST` for this service) + 3. Replace with your custom domain + 4. Replace with your custom domain (should be same as `VIRTUAL_HOST` for this service) + 5. Replace with your custom domain + 6. Replace with your custom domain (should be same as `VIRTUAL_HOST` for this service) + + ## Manually setting up a Neurobagel graph backend The Neurobagel Docker Compose recipe will automatically set up and configure From b20e812defd82b911c8c60a5b309406cf899ef72 Mon Sep 17 00:00:00 2001 From: Alyssa Dai Date: Wed, 13 Nov 2024 16:33:10 -0500 Subject: [PATCH 2/5] make data upload step clearer --- docs/getting_started.md | 6 +++++- docs/maintaining.md | 25 ++++++++++++------------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/docs/getting_started.md b/docs/getting_started.md index d69092d..6f21369 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -101,7 +101,11 @@ cp local_nb_nodes.template.json local_nb_nodes.json you can use `NB_API_QUERY_URL=http://localhost:8080`, where `8080` is the [default host port for the federation API](./config.md#environment-variables). - If you are deploying Neurobagel **on a server for other users**, you must use the IP (and port) or URL intended for your users to access the federation API on the server with. - + +4. (Optional) If you have already [generated Neurobagel JSONLD data files](cli.md), update `LOCAL_GRAPH_DATA` in `.env` to the path containing the data files you wish to add to the graph database. + + Updating the data in the graph can be done at any time. For more information, see [this section](maintaining.md#updating-the-data-in-your-graph). + !!! info This is the minimal configuration you need to make before you can launch Neurobagel. diff --git a/docs/maintaining.md b/docs/maintaining.md index ef89878..147f534 100644 --- a/docs/maintaining.md +++ b/docs/maintaining.md @@ -78,17 +78,16 @@ docker compose --profile full_stack up -d ## Updating the data in your graph -If you have followed the [initial setup](getting_started.md) -and have deployed your Neurobagel node from our Docker Compose recipe, -your node now has a dedicated graph database that stores -the datasets for your node. +The Neurobagel deployment recipe launches a dedicated graph database that stores the datasets for a single node. +The data in this graph database is defined using the [`LOCAL_GRAPH_DATA` environment variable](config.md#environment-variables), and can be changed at any time. + By default, the graph database will only contain an [example dataset called `BIDS synthetic`](https://github.com/neurobagel/recipes/blob/main/data/example_synthetic_pheno-bids.jsonld). -Replacing the existing data in your graph database with your own data (or updated data) is a straightforward process. -Once you have generated or updated the JSONLD files you want to upload, -the process to update the data in your graph is: +If you have followed the [initial setup](getting_started.md) for deploying a Neurobagel node from our Docker Compose recipe, replacing the existing data in your graph database with your own data (or updated data) is a straightforward process. + +Once you have generated or updated the JSONLD files you want to upload, to update the data in your graph: -1. Shut down the Neurobagel node +1. Shut down the Neurobagel node, if it is already running ```bash docker compose --profile full_stack down @@ -96,8 +95,8 @@ the process to update the data in your graph is: (or, replace `full_stack` with the profile you are using) -2. Update the data files in the directory specified by the [`LOCAL_GRAPH_DATA` environment variable](config.md#uploading-data-to-the-graph-store), or simply change the path to a directory containing your JSONLD files. -3. Restart the Neurobagel node +2. Update the data files in the directory specified by the `LOCAL_GRAPH_DATA` variable in `.env`, or simply change the path to a directory containing your JSONLD files. +3. (Re)start the Neurobagel node ```bash docker compose --profile full_stack up -d @@ -150,10 +149,10 @@ This will ensure that if you use the latest version of the Neurobagel CLI to pro Note that if upgrading to a newer version of the data model, **you should regenerate the `.jsonld` files for _all_ datasets in your existing graph**. -### Updating the graph database +### Re-uploading a modified dataset -To allow easy (re-)uploading of the updated `.jsonld` for your dataset(s) to a graph database, make a copy of it in a [central directory on your research data fileserver for storing local Neurobagel `jsonld` datasets](config.md). -Then, follow the steps for [uploading/updating a dataset in the graph database](config.md#uploading-data-to-the-graph-store) (needs to be completed by user with database write access). +To allow easy (re-)uploading of the updated `.jsonld` for your dataset(s) to a graph database, we recommend making a copy of it in a central directory on your research data fileserver for storing local Neurobagel `jsonld` datasets. +Then, simply follow the steps for [uploading/updating a dataset in the graph database](#updating-the-data-in-your-graph). ## Updating your graph backend configuration From 915b30d99541f5d6665f1f431d0c989bfafc641f Mon Sep 17 00:00:00 2001 From: Alyssa Dai Date: Wed, 13 Nov 2024 16:39:35 -0500 Subject: [PATCH 3/5] fix admonition formatting --- docs/config.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/config.md b/docs/config.md index ae26d71..e74d3ee 100644 --- a/docs/config.md +++ b/docs/config.md @@ -467,8 +467,10 @@ you can use the localhost:7200 repositories/my_db DBUSER DBPASSWORD \ ``` !!! warning - To update any _existing_ datasets in your graph database, you can clear the database and reupload all datasets using `add_data_to_graph.sh` following the command above and including the `--clear-data` flag. - Ensure that you also re-upload the Neurobagel vocabulary file `nb_vocab.ttl` following the section below. + To update any _existing_ datasets in your graph database, you can clear the database and reupload all datasets using `add_data_to_graph.sh` following the command above and including the `--clear-data` flag. + + Ensure that you also re-upload the Neurobagel vocabulary file `nb_vocab.ttl` following the section below. + ### Adding vocabulary files to the graph database ??? "Why we need vocabulary files in the graph" From 5a6c7fd5670dad64d4182c7f72643e7faf7181da Mon Sep 17 00:00:00 2001 From: Alyssa Dai Date: Wed, 13 Nov 2024 16:48:03 -0500 Subject: [PATCH 4/5] fix typos --- docs/config.md | 4 ++-- docs/maintaining.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/config.md b/docs/config.md index e74d3ee..3e8458d 100644 --- a/docs/config.md +++ b/docs/config.md @@ -321,9 +321,9 @@ This file adds: 1. Replace with your custom domain 2. Replace with your custom domain (should be same as `VIRTUAL_HOST` for this service) 3. Replace with your custom domain - 4. Replace with your custom domain (should be same as `VIRTUAL_HOST` for this service) + 4. Replace with your custom domain (should be same as `VIRTUAL_HOST` for this service) 5. Replace with your custom domain - 6. Replace with your custom domain (should be same as `VIRTUAL_HOST` for this service) + 6. Replace with your custom domain (should be same as `VIRTUAL_HOST` for this service) ## Manually setting up a Neurobagel graph backend diff --git a/docs/maintaining.md b/docs/maintaining.md index 147f534..36a1cbb 100644 --- a/docs/maintaining.md +++ b/docs/maintaining.md @@ -81,7 +81,7 @@ docker compose --profile full_stack up -d The Neurobagel deployment recipe launches a dedicated graph database that stores the datasets for a single node. The data in this graph database is defined using the [`LOCAL_GRAPH_DATA` environment variable](config.md#environment-variables), and can be changed at any time. -By default, the graph database will only contain an [example dataset called `BIDS synthetic`](https://github.com/neurobagel/recipes/blob/main/data/example_synthetic_pheno-bids.jsonld). +By default, the graph database will only contain an [example dataset called `BIDS synthetic`](https://github.com/neurobagel/recipes/blob/main/data/example_synthetic_pheno-bids-derivatives.jsonld). If you have followed the [initial setup](getting_started.md) for deploying a Neurobagel node from our Docker Compose recipe, replacing the existing data in your graph database with your own data (or updated data) is a straightforward process. From 092498c4353b937356eeb2b69d99b96cfb8f6c01 Mon Sep 17 00:00:00 2001 From: Alyssa Dai Date: Wed, 13 Nov 2024 17:53:18 -0500 Subject: [PATCH 5/5] Clarify phrasing Co-authored-by: Sebastian Urchs --- docs/config.md | 2 +- docs/maintaining.md | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/config.md b/docs/config.md index 3e8458d..d18abd5 100644 --- a/docs/config.md +++ b/docs/config.md @@ -191,7 +191,7 @@ This file adds: docker compose up -d ``` (or, see [here](#available-profiles) to launch a non-default service profile) - 4. Ensure that ports 80 and 443 are reachable for the host machine where your Docker Compose stack is running + 4. Ensure that ports 80 and 443 are open on the host machine where your Docker Compose stack is running ```{ .yaml .annotate title="docker-compose.yml" } services: diff --git a/docs/maintaining.md b/docs/maintaining.md index 36a1cbb..6f52db9 100644 --- a/docs/maintaining.md +++ b/docs/maintaining.md @@ -79,7 +79,9 @@ docker compose --profile full_stack up -d ## Updating the data in your graph The Neurobagel deployment recipe launches a dedicated graph database that stores the datasets for a single node. -The data in this graph database is defined using the [`LOCAL_GRAPH_DATA` environment variable](config.md#environment-variables), and can be changed at any time. +The data in this graph database is loaded from the location specified in the +[`LOCAL_GRAPH_DATA` environment variable](config.md#environment-variables), +and can be changed at any time. By default, the graph database will only contain an [example dataset called `BIDS synthetic`](https://github.com/neurobagel/recipes/blob/main/data/example_synthetic_pheno-bids-derivatives.jsonld).