diff --git a/assets/share/freva/deployment/config/inventory.toml b/assets/share/freva/deployment/config/inventory.toml index 144082ef..4c498846 100644 --- a/assets/share/freva/deployment/config/inventory.toml +++ b/assets/share/freva/deployment/config/inventory.toml @@ -1,4 +1,4 @@ -# This is the "defaut" freva deployment configuration file. +# This is the "default" freva deployment configuration file. # # - The files syntax follows the `toml` markup language (https://toml.io) # - Comments begin with the "#" character @@ -95,6 +95,10 @@ ansible_become_user = "root" ##(Useful for a truely fresh start) (default: False) wipe = false +## Set the path where the permanent database data should be stored. By default +## this is set to /opt/freva//db_service +data_path = "/opt/freva" + ## In case you want to set a custom path to a ansible playbook, ## you can do this here, by default the deployment will use the playbook ## located in the user config directory. @@ -126,6 +130,11 @@ ansible_become_user = "root" ##Indicate whether or not to empty any pre-existing folders/docker volumes. ##(Useful for a truely fresh start) (default: False) wipe = false + +## Set the path where the permanent databrowsr data should be stored. +## By default this is set to /opt/freva +data_path = "/opt/freva" + ## In case you want to set a custom path to a ansible playbook, ## you can do this here, by default the deployment will use the playbook ## located in the user config directory. @@ -216,6 +225,10 @@ project_website = "www.freva.dkrz.de" ##If you need a different user name you can set it here: #ansible_user = "username" +## Set the path where the permanent web data should be stored. By default +## this is set to /opt/freva +data_path = "/opt/freva" + ##Set html colors main_color = "Tomato" border_color = "#6c2e1f" @@ -319,3 +332,9 @@ web_playbook = "" ## Set the become (sudo) user name to change to for installing the services ansible_become_user = "root" + +# Deploy a http reverse proxy. Turn off the deployment (set to false) of an +# reverse proxy doesn't the user that deploys the web doesn't have access to +# port 80 and 443 in this case you will have to deploy the reverse proxy +# yourself. +deploy_web_server = true diff --git a/assets/share/freva/deployment/playbooks/databrowser-server-playbook.yml b/assets/share/freva/deployment/playbooks/databrowser-server-playbook.yml index 6587a81c..8ee9bf0a 100644 --- a/assets/share/freva/deployment/playbooks/databrowser-server-playbook.yml +++ b/assets/share/freva/deployment/playbooks/databrowser-server-playbook.yml @@ -4,17 +4,17 @@ vars: ansible_python_interpreter: "{{ databrowser_ansible_python_interpreter }}" databrowser_name: "{{project_name}}-databrowser" - compose_file: /opt/freva/compose_services/{{databrowser_name}}-compose.yml + compose_file: '{{databrowser_data_path}}/compose_services/{{databrowser_name}}-compose.yml' solr_name: "{{project_name}}-solr" mongo_name: "{{project_name}}-mongo" solr_volumes: - - /opt/freva/{{project_name}}/solr_service:/var/solr/data:z - - /opt/freva/freva-service-config/solr/managed-schema.xml:/opt/solr/managed-schema.xml:z - - /opt/freva/freva-service-config/solr/create_cores.sh:/docker-entrypoint-initdb.d/create_cores.sh:z - - /opt/freva/freva-service-config/solr/synonyms.txt:/opt/solr/synonyms.txt:z - - /opt/freva/freva-service-config/solr/daily_backup.sh:/usr/local/bin/daily_backup:z + - '{{databrowser_data_path}}/{{project_name}}/solr_service:/var/solr/data:z' + - '{{databrowser_data_path}}/freva-service-config/solr/managed-schema.xml:/opt/solr/managed-schema.xml:z' + - '{{databrowser_data_path}}/freva-service-config/solr/create_cores.sh:/docker-entrypoint-initdb.d/create_cores.sh:z' + - '{{databrowser_data_path}}/freva-service-config/solr/synonyms.txt:/opt/solr/synonyms.txt:z' + - '{{databrowser_data_path}}/freva-service-config/solr/daily_backup.sh:/usr/local/bin/daily_backup:z' mongo_volumes: - - /opt/freva/{{project_name}}/databrowser/stats:/data/db:z + - '{{databrowser_data_path}}/{{project_name}}/databrowser/stats:/data/db:z' ansible_become_user: "{{ databrowser_ansible_become_user | default('root') }}" use_become: "{{ databrowser_ansible_become_user is defined and databrowser_ansible_become_user != '' }}" tasks: @@ -58,20 +58,20 @@ file: state: absent force: true - path: /opt/freva/freva-service-config/ + path: '{{databrowser_data_path}}/freva-service-config/' become: "{{use_become}}" - name: Cleaning existing directory structure file: path: "{item}" state: absent with_items: - - "/opt/freva/{{ project_name }}/solr_service" - - "/opt/freva/{{ project_name }}/databrowser" + - "{{databrowser_data_path}}/{{ project_name }}/solr_service" + - "{{databrowser_data_path}}/{{ project_name }}/databrowser" when: databrowser_wipe == true become: "{{use_become}}" - name: Creating solr directory structure file: - path: /opt/freva/{{ project_name }}/solr_service + path: '{{databrowser_data_path}}/{{ project_name }}/solr_service' state: directory owner: 8983 group: 8983 @@ -79,20 +79,20 @@ become: "{{use_become}}" - name: Creating compose directory structure file: - path: /opt/freva/compose_services + path: '{{databrowser_data_path}}/compose_services' state: directory recurse: true become: "{{use_become}}" - name: Creating directory structure file: - path: /opt/freva/{{ project_name }}/databrowser + path: '{{databrowser_data_path}}/{{ project_name }}/databrowser' state: directory recurse: true become: "{{use_become}}" - name: Getting additional configurations git: repo: https://github.com/FREVA-CLINT/freva-service-config.git - dest: /opt/freva/freva-service-config + dest: '{{databrowser_data_path}}/freva-service-config' update: true become: "{{use_become}}" - name: Copy systemd files diff --git a/assets/share/freva/deployment/playbooks/db-server-playbook.yml b/assets/share/freva/deployment/playbooks/db-server-playbook.yml index 034f46d1..fc87069c 100644 --- a/assets/share/freva/deployment/playbooks/db-server-playbook.yml +++ b/assets/share/freva/deployment/playbooks/db-server-playbook.yml @@ -3,14 +3,14 @@ vars: ansible_python_interpreter: "{{ db_ansible_python_interpreter }}" - compose_file: /opt/freva/compose_services/{{db_name}}-compose.yml + compose_file: '{{db_data_path}}/compose_services/{{db_name}}-compose.yml' db_volumes: - - /opt/freva/freva-service-config/mysql/create_tables.sql:/docker-entrypoint-initdb.d/002_create_tables.sql:z - - /opt/freva/freva-service-config/mysql/daily_backup.sh:/usr/local/bin/daily_backup:z - - /opt/freva/{{project_name}}/db_service:/var/lib/mysql:z + - '{{db_data_path}}/freva-service-config/mysql/create_tables.sql:/docker-entrypoint-initdb.d/002_create_tables.sql:z' + - '{{db_data_path}}/freva-service-config/mysql/daily_backup.sh:/usr/local/bin/daily_backup:z' + - '{{db_data_path}}/{{project_name}}/db_service:/var/lib/mysql:z' docker_cmd: > --network {{ project_name }} -v - /opt/freva/{{project_name}}/db_service:/var/lib/mysql:z + {{db_data_path}}/{{project_name}}/db_service:/var/lib/mysql:z -e HOST={{ db_host }} -e NUM_BACKUPS=7 -e PROJECT={{ project_name }} @@ -19,15 +19,15 @@ -e MYSQL_DATABASE={{db}} -e BACKUP_DIR=/var/lib/mysql/backup -p {{ db_port }}:3306 - -v /opt/freva/freva-service-config/mysql/create_tables.sql:/docker-entrypoint-initdb.d/002_create_tables.sql:z - -v /opt/freva/freva-service-config/mysql/daily_backup.sh:/usr/local/bin/daily_backup:z + -v '{{db_data_path}}/freva-service-config/mysql/create_tables.sql:/docker-entrypoint-initdb.d/002_create_tables.sql:z' + -v '{{db_data_path}}/freva-service-config/mysql/daily_backup.sh:/usr/local/bin/daily_backup:z' --rm --name {{db_name}} -e MYSQL_ROOT_PASSWORD={{ root_passwd }} -t docker.io/mariadb:latest skip_tables_cmd: > --network {{ project_name }} -v - /opt/freva/{{project_name}}/db_service:/var/lib/mysql:z + '{{db_data_path}}/{{project_name}}/db_service:/var/lib/mysql:z' -e HOST={{ db_host }} -e NUM_BACKUPS=7 -e PROJECT={{ project_name }} @@ -38,8 +38,8 @@ -p {{ db_port }}:3306 -e MYSQL_ROOT_PASSWORD={{ root_passwd }} --name {{db_name}} - -v /opt/freva/freva-service-config/mysql/create_tables.sql:/docker-entrypoint-initdb.d/002_create_tables.sql:z - -v /opt/freva/freva-service-config/mysql/daily_backup.sh:/usr/local/bin/daily_backup:z + -v '{{db_data_path}}/freva-service-config/mysql/create_tables.sql:/docker-entrypoint-initdb.d/002_create_tables.sql:z' + -v '{{db_data_path}}/freva-service-config/mysql/daily_backup.sh:/usr/local/bin/daily_backup:z' -v /tmp/reset_root_pw.sh:/tmp/reset_root_pw.sh:z -t docker.io/mariadb:latest mariadbd-safe --skip-grant-tables continer_name: "{{ db_name }}" @@ -74,7 +74,7 @@ file: state: absent force: true - path: "/opt/freva/freva-service-config/" + path: "{{db_data_path}}/freva-service-config/" become: "{{use_become}}" - name: Pulling container become: "{{use_become}}" @@ -86,13 +86,13 @@ become: "{{use_become}}" - name: Cleaning existing directory structure file: - path: /opt/freva/{{ project_name }}/db_service + path: '{{db_data_path}}/{{ project_name }}/db_service' state: absent become: "{{use_become}}" when: db_wipe == true - name: Creating directory structure file: - path: /opt/freva/{{ project_name }}/db_service + path: '{{db_data_path}}/{{ project_name }}/db_service' state: directory recurse: true group: 999 @@ -113,7 +113,7 @@ become: "{{use_become}}" git: repo: https://github.com/FREVA-CLINT/freva-service-config.git - dest: /opt/freva/freva-service-config + dest: '{{db_data_path}}/freva-service-config' update: true - name: Preparing the root password reset I become: "{{use_become}}" @@ -129,14 +129,14 @@ - pause: seconds=2 - name: Resetting the root password become: "{{use_become}}" - shell: /tmp/docker-or-podman exec -it {{db_name}} bash /tmp/reset_root_pw.sh + shell: /tmp/docker-or-podman exec {{db_name}} bash /tmp/reset_root_pw.sh - name: Deleting temp. container become: "{{use_become}}" shell: | /tmp/docker-or-podman rm -f {{db_name}} - name: Creating compose directory structure file: - path: /opt/freva/compose_services + path: '{{db_data_path}}/compose_services' state: directory recurse: true become: "{{use_become}}" @@ -144,7 +144,7 @@ become: "{{use_become}}" template: src: "{{ asset_dir }}/playbooks/db-server-compose-template.yml" - dest: "/opt/freva/compose_services/{{db_name}}-compose.yml" + dest: "{{db_data_path}}/compose_services/{{db_name}}-compose.yml" - name: Creating system services become: "{{use_become}}" shell: | @@ -158,7 +158,7 @@ - name: Creating potentially missing tables become: "{{use_become}}" shell: | - /tmp/docker-or-podman exec -it {{db_name}} /bin/sh -c 'mariadb -u root -p{{root_passwd}} {{db}} < /docker-entrypoint-initdb.d/002_create_tables.sql' + /tmp/docker-or-podman exec {{db_name}} /bin/sh -c 'mariadb -u root -p{{root_passwd}} {{db}} < /docker-entrypoint-initdb.d/002_create_tables.sql' - name: Deleting auxillary files become: "{{use_become}}" file: diff --git a/assets/share/freva/deployment/playbooks/server-map-playbook.yml b/assets/share/freva/deployment/playbooks/server-map-playbook.yml index bf88d027..bec40314 100644 --- a/assets/share/freva/deployment/playbooks/server-map-playbook.yml +++ b/assets/share/freva/deployment/playbooks/server-map-playbook.yml @@ -4,7 +4,7 @@ vars: docker_cmd: > run - -v /opt/freva/server-map:/var/freva/:z + -v '{{data_path}}:/var/freva/:z' --dns 8.8.8.8 -p {{ port }}:5008 --name freva-map @@ -33,7 +33,7 @@ when: wipe == true - name: Cleaning existing directory structure file: - path: /opt/freva/server-map + path: '{{data_path}}' state: absent when: wipe == true - name: Copy systemd files @@ -45,7 +45,7 @@ copy: src="{{ asset_dir }}/servers" dest=/tmp - name: Creating directory structure file: - path: /opt/freva/server-map + path: '{{data_path}}' state: directory owner: 9999 group: 9999 diff --git a/assets/share/freva/deployment/playbooks/solr-server-playbook.yml b/assets/share/freva/deployment/playbooks/solr-server-playbook.yml index fb9b0b30..d1721830 100644 --- a/assets/share/freva/deployment/playbooks/solr-server-playbook.yml +++ b/assets/share/freva/deployment/playbooks/solr-server-playbook.yml @@ -9,11 +9,11 @@ -e NUM_BACKUPS=7 -e SOLR_HEAP={{solr_mem}} --rm - -v /opt/freva/{{project_name}}/solr_service:/var/solr/data:z - -v /opt/freva/freva-service-config/solr/managed-schema.xml:/opt/solr/managed-schema.xml:z - -v /opt/freva/freva-service-config/solr/create_cores.sh:/docker-entrypoint-initdb.d/create_cores.sh:z - -v /opt/freva/freva-service-config/solr/synonyms.txt:/opt/solr/synonyms.txt:z - -v /opt/freva/freva-service-config/solr/daily_backup.sh:/usr/local/bin/daily_backup:z + -v '{{solr_data_path}}/{{project_name}}/solr_service:/var/solr/data:z' + -v '{{solr_data_path}}/freva-service-config/solr/managed-schema.xml:/opt/solr/managed-schema.xml:z' + -v '{{solr_data_path}}/freva-service-config/solr/create_cores.sh:/docker-entrypoint-initdb.d/create_cores.sh:z' + -v '{{solr_data_path}}/freva-service-config/solr/synonyms.txt:/opt/solr/synonyms.txt:z' + -v '{{solr_data_path}}/freva-service-config/solr/daily_backup.sh:/usr/local/bin/daily_backup:z' -p {{ solr_port }}:8983 -t --name {{solr_name}} -t solr:latest @@ -47,7 +47,7 @@ file: state: absent force: true - path: "/opt/freva/freva-service-config/" + path: "{{solr_data_path}}/freva-service-config/" become: "{{use_become}}" - name: Creating docker network shell: > @@ -57,13 +57,13 @@ become: "{{use_become}}" - name: Cleaning existing directory structure file: - path: /opt/freva/{{ project_name }}/solr_service + path: '{{solr_data_path}}/{{ project_name }}/solr_service' state: absent when: solr_wipe == true become: "{{use_become}}" - name: Creating directory structure file: - path: /opt/freva/{{ project_name }}/solr_service + path: '{{solr_data_path}}/{{ project_name }}/solr_service' state: directory owner: 8983 group: 8983 @@ -72,7 +72,7 @@ - name: Getting additional configurations git: repo: https://github.com/FREVA-CLINT/freva-service-config.git - dest: /opt/freva/freva-service-config + dest: '{{solr_data_path}}/freva-service-config' update: true become: "{{use_become}}" - name: Copy systemd files diff --git a/assets/share/freva/deployment/playbooks/vault-server-playbook.yml b/assets/share/freva/deployment/playbooks/vault-server-playbook.yml index e678b679..28934311 100644 --- a/assets/share/freva/deployment/playbooks/vault-server-playbook.yml +++ b/assets/share/freva/deployment/playbooks/vault-server-playbook.yml @@ -4,10 +4,11 @@ vars: ansible_python_interpreter: "{{ db_ansible_python_interpreter }}" continer_name: "{{ vault_name }}" - compose_file: /opt/freva/compose_services/{{vault_name}}-compose.yml + compose_file: "{{vault_data_path}}/compose_services/{{vault_name}}-compose.yml" vault_volumes: - - /opt/freva/{{project_name}}/vault_service/config:/data:z - - /opt/freva/{{project_name}}/vault_service/files:/vault/file:z + - '{{vault_data_path}}/{{project_name}}/vault_service/config:/data:z' + - '{{vault_data_path}}/{{project_name}}/vault_service/files:/vault/file:z' + - /home/wilfred/workspace/deployment/assets/share/freva/deployment/vault/runserver.py:/bin/runserver:z use_become: "{{ vault_ansible_become_user is defined and vault_ansible_become_user != '' }}" tasks: - name: Copying docker/podman wrapper script @@ -31,13 +32,13 @@ systemctl stop {{vault_name}}; systemctl disable {{vault_name}}; /tmp/docker-or-podman rm {{vault_name}}; - /tmp/docker-or-podman rmi -f {{vault_name}} ghcr.io/freva-clint/freva-deployment/vault; + # /tmp/docker-or-podman rmi -f {{vault_name}} ghcr.io/freva-clint/freva-deployment/vault; exit 0 ignore_errors: true become: "{{use_become}}" - - name: Pulling container - become: "{{use_become}}" - shell: /tmp/docker-or-podman pull ghcr.io/freva-clint/freva-deployment/vault:latest + # - name: Pulling container + # become: "{{use_become}}" + # shell: /tmp/docker-or-podman pull ghcr.io/freva-clint/freva-deployment/vault:latest - name: Copying auxillary files to target machine copy: src="{{ asset_dir }}/vault" dest=/tmp - name: Copying systemd files @@ -50,17 +51,17 @@ state: directory path: "{{ item }}" with_items: - - "/opt/freva/{{ project_name }}/vault_service/config" - - "/opt/freva/{{ project_name }}/vault_service/files" + - "{{vault_data_path}}/{{ project_name }}/vault_service/config" + - "{{vault_data_path}}/{{ project_name }}/vault_service/files" become: "{{use_become}}" - name: Copying public key file to target machine copy: src: "{{ vault_keyfile }}" - dest: "/opt/freva/{{project_name}}/vault_service/config/freva.crt" + dest: "{{vault_data_path}}/{{project_name}}/vault_service/config/freva.crt" become: "{{use_become}}" - name: Creating compose directory structure file: - path: /opt/freva/compose_services + path: '{{vault_data_path}}/compose_services' state: directory recurse: true become: "{{use_become}}" @@ -68,7 +69,7 @@ become: "{{use_become}}" template: src: "{{ asset_dir }}/playbooks/vault-server-compose-template.yml" - dest: "/opt/freva/compose_services/{{vault_name}}-compose.yml" + dest: "{{vault_data_path}}/compose_services/{{vault_name}}-compose.yml" - name: Creating system services become: "{{use_become}}" shell: | @@ -76,14 +77,12 @@ when: systemctl.stat.exists == true - pause: seconds=20 - name: Deleting existing infrastructure - shell: > - /tmp/docker-or-podman exec -it {{ vault_name }} - vault kv delete kv/read-eval + shell: /tmp/docker-or-podman exec {{ vault_name }} vault kv delete kv/read-eval become: "{{use_become}}" - name: Inserting server infrastructure become: "{{use_become}}" shell: > - /tmp/docker-or-podman exec -it {{ vault_name }} + /tmp/docker-or-podman exec {{ vault_name }} vault kv put kv/read-eval db.container={{ vault_name }} db.host={{ vault_host }} db.port={{ db_port }} diff --git a/assets/share/freva/deployment/playbooks/web-server-playbook.yml b/assets/share/freva/deployment/playbooks/web-server-playbook.yml index 431e4d46..c5748b95 100644 --- a/assets/share/freva/deployment/playbooks/web-server-playbook.yml +++ b/assets/share/freva/deployment/playbooks/web-server-playbook.yml @@ -12,12 +12,12 @@ - name: Deleting existing email secrets become: "{{use_become}}" shell: > - /tmp/docker-or-podman exec -it {{ vault_name }} + /tmp/docker-or-podman exec {{ vault_name }} vault kv delete kv/email - name: Inserting email secrets become: "{{use_become}}" shell: > - /tmp/docker-or-podman exec -it {{ vault_name }} + /tmp/docker-or-podman exec {{ vault_name }} vault kv put kv/email username={{vault_email_user}} password='{{vault_email_password}}' - name: Deleting tmporary files @@ -60,20 +60,20 @@ vars: ansible_python_interpreter: "{{ web_ansible_python_interpreter }}" - service_dir: /opt/freva/{{project_name}}/web_service - compose_file: /opt/freva/compose_services/{{web_name}}-compose.yml + service_dir: '{{web_data_path}}/{{project_name}}/web_service' + compose_file: '{{web_data_path}}/compose_services/{{web_name}}-compose.yml' web_volumes: - "{{ core_root_dir }}:{{ core_root_dir }}:ro" - "{{ core_scheduler_output_dir}}:{{ core_scheduler_output_dir }}:ro" - - "/opt/freva/{{project_name}}/web_service/static:/opt/freva_web/static:z" + - "{{web_data_path}}/{{project_name}}/web_service/static:{{web_data_path}}_web/static:z" - "{{core_base_dir_location}}:{{core_base_dir_location}}:ro" apache_volumes: - - "/opt/freva/{{project_name}}/web_service/freva_web.conf:/usr/local/apache2/conf/httpd.conf:z" - - "/opt/freva/{{project_name}}/web_service/server-cert.crt:/etc/ssl/certs/server-cert.crt:z" - - "/opt/freva/{{project_name}}/web_service/server-key.key:/etc/ssl/private/server-key.key:z" - - "/opt/freva/{{project_name}}/web_service/cacert.pem:/etc/ssl/certs/cacert.pem:z" - - "/opt/freva/{{project_name}}/web_service/static:/srv/static:z" - - "/opt/freva/{{project_name}}/web_service/prepare-httpd:/usr/local/bin/prepare-httpd:z" + - "{{web_data_path}}/{{project_name}}/web_service/freva_web.conf:/usr/local/apache2/conf/httpd.conf:z" + - "{{web_data_path}}/{{project_name}}/web_service/server-cert.crt:/etc/ssl/certs/server-cert.crt:z" + - "{{web_data_path}}/{{project_name}}/web_service/server-key.key:/etc/ssl/private/server-key.key:z" + - "{{web_data_path}}/{{project_name}}/web_service/cacert.pem:/etc/ssl/certs/cacert.pem:z" + - "{{web_data_path}}/{{project_name}}/web_service/static:/srv/static:z" + - "{{web_data_path}}/{{project_name}}/web_service/prepare-httpd:/usr/local/bin/prepare-httpd:z" - "{{ core_preview_path }}:/srv/static/preview:z" docker_apache_cmd: > --rm @@ -145,11 +145,11 @@ become: "{{use_become}}" copy: src: "{{ asset_dir }}/scripts/prepare-httpd" - dest: "/opt/freva/{{project_name}}/web_service/prepare-httpd" + dest: "{{web_data_path}}/{{project_name}}/web_service/prepare-httpd" mode: "0775" - name: Creating compose directory structure file: - path: /opt/freva/compose_services + path: '{{web_data_path}}/compose_services' state: directory recurse: true become: "{{use_become}}" diff --git a/src/freva_deployment/cli/_deploy.py b/src/freva_deployment/cli/_deploy.py index ea5340c6..4ea1925e 100644 --- a/src/freva_deployment/cli/_deploy.py +++ b/src/freva_deployment/cli/_deploy.py @@ -53,6 +53,13 @@ def parse_args(argv: list[str] | None) -> argparse.Namespace: ap.add_argument( "-v", "--verbose", action="count", help="Verbosity level", default=0 ) + ap.add_argument( + "-l", + "--local", + help="Deploy services on the local machine, debug purpose.", + action="store_true", + default=False, + ) ap.add_argument( "-V", "--version", @@ -74,7 +81,9 @@ def cli(argv: list[str] | None = None) -> None: steps=args.steps, config_file=args.config, ) as DF: - DF.play(server_map, args.ask_pass, args.verbose) + DF.play( + server_map, args.ask_pass, args.verbose, local_debug=args.local + ) if __name__ == "__main__": diff --git a/src/freva_deployment/cli/_server_map.py b/src/freva_deployment/cli/_server_map.py index 7cb5595b..cb8168f6 100644 --- a/src/freva_deployment/cli/_server_map.py +++ b/src/freva_deployment/cli/_server_map.py @@ -64,6 +64,12 @@ def parse_args(args: list[str]) -> argparse.Namespace: type=Path, help="Path to the default python3 interpreter on the target machine.", ) + app.add_argument( + "--data-path", + default="/opt/freva/server-map", + type=Path, + help="Path that is used to save consistent data.", + ) app.add_argument( "-v", "--verbose", action="count", help="Verbosity level", default=0 ) @@ -85,6 +91,7 @@ def parse_args(args: list[str]) -> argparse.Namespace: port: {port} asset_dir: {asset_dir} wipe: {wipe} + data_path: {data_path} """ @@ -114,6 +121,7 @@ def cli(argv: list[str] | None = None) -> None: port=argp.port, asset_dir=asset_dir, wipe=str(argp.wipe).lower(), + data_path=argp.data_path, ) logger.debug(inventory) with open(inventory_file.name, "w") as f_obj: diff --git a/src/freva_deployment/deploy.py b/src/freva_deployment/deploy.py index 14e53a47..871fe57a 100644 --- a/src/freva_deployment/deploy.py +++ b/src/freva_deployment/deploy.py @@ -11,7 +11,7 @@ from pathlib import Path from subprocess import run from tempfile import TemporaryDirectory -from typing import Any +from typing import Any, cast from urllib.parse import urlparse import tomlkit @@ -60,7 +60,7 @@ def __init__( config_file: Path | str | None = None, ) -> None: self._config_keys: list[str] = [] - self.master_pass: str = "" + self.master_pass: str = "Freva4all!" self.email_password: str = "" self._td: TemporaryDirectory = TemporaryDirectory(prefix="inventory") self.inventory_file: Path = Path(self._td.name) / "inventory.yaml" @@ -70,7 +70,7 @@ def __init__( self.web_conf_file: Path = Path(self._td.name) / "freva_web.toml" self.apache_config: Path = Path(self._td.name) / "freva_web.conf" self._db_pass: str = "" - self._steps = steps or ["services", "core", "web"] + self._steps = steps or ["db", "databrowser", "web"] self._inv_tmpl = Path(config_file or config_dir / "inventory.toml") self._cfg_tmpl = self.aux_dir / "evaluation_system.conf.tmpl" self.cfg = self._read_cfg() @@ -133,6 +133,12 @@ def _prep_db(self) -> None: self.cfg["db"]["config"]["root_passwd"] = self.master_pass self.cfg["db"]["config"]["passwd"] = self.db_pass self.cfg["db"]["config"]["keyfile"] = self.public_key_file + data_path = Path( + cast(str, self.cfg["db"]["config"].get("data_path", "/opt/freva")) + ) + self.cfg["db"]["config"]["data_path"] = str( + data_path.absolute().expanduser() + ) for key in ("name", "user", "db"): self.cfg["db"]["config"][key] = ( self.cfg["db"]["config"].get(key) or "freva" @@ -158,6 +164,17 @@ def _prep_databrowser(self, prep_web=True) -> None: ) self.cfg["databrowser"]["config"]["root_passwd"] = self.master_pass self.cfg["databrowser"]["config"].pop("core", None) + data_path = Path( + cast( + str, + self.cfg["databrowser"]["config"].get( + "data_path", "/opt/freva" + ), + ) + ) + self.cfg["databrowser"]["config"]["data_path"] = str( + data_path.absolute().expanduser() + ) self.playbooks["databrowser"] = self.cfg["databrowser"]["config"].get( "databrowser_playbook" ) @@ -222,8 +239,6 @@ def _prep_core(self) -> None: def _prep_web(self, ask_pass: bool = True) -> None: """prepare the web deployment.""" self._config_keys.append("web") - # if ask_pass: - # self._prep_vault() self.playbooks["web"] = self.cfg["web"]["config"].get("web_playbook") self.cfg["web"]["config"].setdefault("ansible_become_user", "root") self._prep_core() @@ -232,6 +247,13 @@ def _prep_web(self, ask_pass: bool = True) -> None: f'{self.cfg["databrowser"]["config"]["databrowser_port"]}' ) self.cfg["web"]["config"]["databrowser_host"] = databrowser_host + self.cfg["web"]["config"].setdefault("deploy_web_server", True) + data_path = Path( + cast(str, self.cfg["web"]["config"].get("data_path", "/opt/freva")) + ) + self.cfg["web"]["config"]["data_path"] = str( + data_path.absolute().expanduser() + ) admin = self.cfg["core"]["config"]["admins"] if not isinstance(admin, str): self.cfg["web"]["config"]["admin"] = admin[0] @@ -589,6 +611,7 @@ def play( server_map: str | None = None, ask_pass: bool = True, verbosity: int = 0, + local_debug: bool = False, ) -> None: """Play the ansible playbook. @@ -603,6 +626,8 @@ def play( ssh key verbosity: int, default: 0 Verbosity level, default 0 + local_debug: bool, default: False + Run deployment only on local machine, debug mode. """ inventory = self.parse_config() self.create_eval_config() @@ -626,6 +651,8 @@ def play( ) if ask_pass: cmd += " --ask-pass" + if local_debug: + cmd += " -c local" env = os.environ.copy() env["PATH"] += f":{AssetDir.get_dirs(user=True, key='scripts')}" try: diff --git a/src/freva_deployment/ui/deployment_tui/deploy_forms.py b/src/freva_deployment/ui/deployment_tui/deploy_forms.py index 1f1a0ecb..17fbe0bd 100644 --- a/src/freva_deployment/ui/deployment_tui/deploy_forms.py +++ b/src/freva_deployment/ui/deployment_tui/deploy_forms.py @@ -85,7 +85,9 @@ def _add_widgets(self) -> None: max_height=2, value=cfg.get("install", True), editable=True, - name=(f"{self.num}Install a new Freva anaconda environment?"), + name=( + f"{self.num}Install a new Freva anaconda environment?" + ), scroll_exit=True, ), True, @@ -121,7 +123,9 @@ def _add_widgets(self) -> None: self.add_widget_intelligent( npyscreen.TitleCombo, name=f"{self.num}Workload manger", - value=self.scheduler_index(cast(str, cfg.get("scheduler_system"))), + value=self.scheduler_index( + cast(str, cfg.get("scheduler_system")) + ), values=self.scheduler_systems, ), True, @@ -204,7 +208,9 @@ def _add_widgets(self) -> None: self.add_widget_intelligent( npyscreen.TitleFilename, name=f"{self.num}Python path on remote machine:", - value=cfg.get("ansible_python_interpreter", "/usr/bin/python3"), + value=cfg.get( + "ansible_python_interpreter", "/usr/bin/python3" + ), ), False, ), @@ -251,6 +257,17 @@ def _add_widgets(self) -> None: cfg[key] = [v.strip() for v in value.split(",") if v.strip()] logger.warning(key, cfg[key]) self.input_fields: dict[str, tuple[npyscreen.TitleText, bool]] = dict( + deploy_web_server=( + self.add_widget_intelligent( + npyscreen.RoundCheckBox, + max_height=2, + value=cfg.get("deployed_web_server", True), + editable=True, + name=(f"{self.num}Set up the http reverse proxy server?"), + scroll_exit=True, + ), + True, + ), hosts=( self.add_widget_intelligent( npyscreen.TitleText, @@ -259,6 +276,19 @@ def _add_widgets(self) -> None: ), True, ), + data_path=( + self.add_widget_intelligent( + npyscreen.TitleText, + name=( + f"{self.num}Set the path where the permanent web " + "data should be stored" + ), + value=str( + Path(cast(str, cfg.get("data_path", "/opt/freva"))) + ), + ), + True, + ), project_website=( self.add_widget_intelligent( npyscreen.TitleText, @@ -376,7 +406,9 @@ def _add_widgets(self) -> None: self.add_widget_intelligent( npyscreen.TitleText, name=f"{self.num}A brief describtion of the project:", - value=cfg.get("homepage_heading", "Lorem ipsum dolor sit amet"), + value=cfg.get( + "homepage_heading", "Lorem ipsum dolor sit amet" + ), ), True, ), @@ -541,7 +573,9 @@ def _add_widgets(self) -> None: ldap_model=( self.add_widget_intelligent( npyscreen.TitleText, - name=(f"{self.num}Ldap tools class to be used for authentication."), + name=( + f"{self.num}Ldap tools class to be used for authentication." + ), value=cfg.get("ldap_model", "MiklipUserInformation"), ), True, @@ -572,7 +606,9 @@ def _add_widgets(self) -> None: self.add_widget_intelligent( npyscreen.TitleFilename, name=f"{self.num}Pythonpath on remote machine:", - value=cfg.get("ansible_python_interpreter", "/usr/bin/python3"), + value=cfg.get( + "ansible_python_interpreter", "/usr/bin/python3" + ), ), False, ), @@ -647,6 +683,19 @@ def _add_widgets(self) -> None: ), True, ), + data_path=( + self.add_widget_intelligent( + npyscreen.TitleText, + name=( + f"{self.num}Set the path where the permanent " + "database data should be stored" + ), + value=str( + Path(cast(str, cfg.get("data_path", "/opt/freva"))) + ), + ), + True, + ), db_playbook=( self.add_widget_intelligent( npyscreen.TitleFilename, @@ -684,7 +733,9 @@ def _add_widgets(self) -> None: self.add_widget_intelligent( npyscreen.TitleFilename, name=f"{self.num}Pythonpath on remote machine:", - value=cfg.get("ansible_python_interpreter", "/usr/bin/python3"), + value=cfg.get( + "ansible_python_interpreter", "/usr/bin/python3" + ), ), False, ), @@ -765,6 +816,19 @@ def _add_widgets(self) -> None: ), True, ), + data_path=( + self.add_widget_intelligent( + npyscreen.TitleText, + name=( + f"{self.num}Set the path where the permanent " + "databrowser data should be stored" + ), + value=str( + Path(cast(str, cfg.get("data_path", "/opt/freva"))) + ), + ), + True, + ), databrowser_playbook=( self.add_widget_intelligent( npyscreen.TitleFilename, @@ -791,7 +855,9 @@ def _add_widgets(self) -> None: self.add_widget_intelligent( npyscreen.TitleFilename, name=f"{self.num}Pythonpath on remote machine:", - value=cfg.get("ansible_python_interpreter", "/usr/bin/python3"), + value=cfg.get( + "ansible_python_interpreter", "/usr/bin/python3" + ), ), False, ), @@ -822,7 +888,9 @@ def on_ok(self) -> None: self.parentApp.thread_stop.set() if not self.project_name.value: - npyscreen.notify_confirm("You have to set a project name", title="ERROR") + npyscreen.notify_confirm( + "You have to set a project name", title="ERROR" + ) return missing_form: None | str = self.parentApp.check_missing_config() if missing_form: @@ -837,7 +905,9 @@ def on_ok(self) -> None: self.parentApp.get_save_file(self.inventory_file.value or None) ) for key_type, keyfile in cert_files.items(): - key_file = Path(get_current_file_dir(save_file.parent, str(keyfile))) + key_file = Path( + get_current_file_dir(save_file.parent, str(keyfile)) + ) for step, deploy_form in self.parentApp._forms.items(): if not keyfile or not Path(key_file).is_file(): if ( @@ -845,11 +915,11 @@ def on_ok(self) -> None: and step in self.parentApp.steps ): if keyfile: + msg = f"{key_type} certificate file `{key_file}` must exist." + else: msg = ( - f"{key_type} certificate file `{key_file}` must exist." + f"You must give a {key_type} certificate file." ) - else: - msg = f"You must give a {key_type} certificate file." npyscreen.notify_confirm(msg, title="ERROR") return @@ -912,7 +982,9 @@ def _add_widgets(self) -> None: ) self.server_map = self.add_widget_intelligent( npyscreen.TitleText, - name=(f"{self.num}Hostname of the service mapping the freva server arch."), + name=( + f"{self.num}Hostname of the service mapping the freva server arch." + ), value=self.parentApp._read_cache("server_map", ""), ) self.public_keyfile = self.add_widget_intelligent( diff --git a/src/freva_deployment/utils.py b/src/freva_deployment/utils.py index 74b3f802..6bb2307c 100644 --- a/src/freva_deployment/utils.py +++ b/src/freva_deployment/utils.py @@ -66,12 +66,8 @@ class AssetDir: this_module = "freva_deployment" def __init__(self): - self._user_asset_dir = ( - Path(appdirs.user_data_dir()) / "freva" / "deployment" - ) - self._user_config_dir = ( - Path(appdirs.user_config_dir()) / "freva" / "deployment" - ) + self._user_asset_dir = Path(appdirs.user_data_dir()) / "freva" / "deployment" + self._user_config_dir = Path(appdirs.user_config_dir()) / "freva" / "deployment" @staticmethod def get_dirs(user: bool = True, key: str = "data") -> Path: @@ -116,9 +112,7 @@ def config_dir(self): inventory_file.unlink() except FileNotFoundError: pass - shutil.copy2( - self.asset_dir / "config" / "inventory.toml", inventory_file - ) + shutil.copy2(self.asset_dir / "config" / "inventory.toml", inventory_file) return self._user_config_dir @property @@ -186,14 +180,12 @@ def _create_new_config(inp_file: Path) -> Path: config["databrowser"] = config.pop("solr") for key in ("port", "mem"): if key in config["databrowser"]["config"]: - config["databrowser"]["config"][f"solr_{key}"] = config[ - "databrowser" - ]["config"].pop(key) + config["databrowser"]["config"][f"solr_{key}"] = config["databrowser"][ + "config" + ].pop(key) _update_config(config_tmpl, config) if create_backup: - inp_file.with_suffix(inp_file.suffix + ".bck").write_text( - inp_file.read_text() - ) + inp_file.with_suffix(inp_file.suffix + ".bck").write_text(inp_file.read_text()) inp_file.write_text(tomlkit.dumps(config_tmpl)) return inp_file @@ -265,9 +257,7 @@ def set_log_level(verbosity: int) -> None: logger.setLevel(max(logging.INFO - 10 * verbosity, logging.DEBUG)) -def get_setup_for_service( - service: str, setups: list[ServiceInfo] -) -> tuple[str, str]: +def get_setup_for_service(service: str, setups: list[ServiceInfo]) -> tuple[str, str]: """Get the setup of a service configuration.""" for setup in setups: if setup.name == service: @@ -280,9 +270,7 @@ def read_db_credentials( ) -> dict[str, str]: """Read database config.""" with cert_file.open() as f_obj: - key = "".join( - [k.strip() for k in f_obj.readlines() if not k.startswith("-")] - ) + key = "".join([k.strip() for k in f_obj.readlines() if not k.startswith("-")]) sha = hashlib.sha512(key.encode()).hexdigest() url = f"http://{db_host}:{port}/vault/data/{sha}" return requests.get(url).json() @@ -362,9 +350,7 @@ def get_email_credentials() -> tuple[str, str]: ) RichConsole.print(msg) username = Prompt.ask("[green b]Username[/] for mail server") - password = Prompt.ask( - "[green b]Password[/] for mail server", password=True - ) + password = Prompt.ask("[green b]Password[/] for mail server", password=True) return username, password @@ -399,9 +385,7 @@ def _create_passwd(min_characters: int, msg: str = "") -> str: if not re.search(check, master_pass): is_ok = False break - is_safe: bool = ( - len([True for c in "[_@$#$%^&*-!]" if c in master_pass]) > 0 - ) + is_safe: bool = len([True for c in "[_@$#$%^&*-!]" if c in master_pass]) > 0 if is_ok is False or is_safe is False: raise ValueError( ( @@ -411,9 +395,7 @@ def _create_passwd(min_characters: int, msg: str = "") -> str: "- have at least one special special character." ) ) - master_pass_2 = Prompt.ask( - "[bold green]re-enter[/] master password", password=True - ) + master_pass_2 = Prompt.ask("[bold green]re-enter[/] master password", password=True) if master_pass != master_pass_2: raise ValueError("Passwords do not match") return master_pass