Databrowser api

.github/workflows/build_job.yml

@@ -50,7 +50,7 @@ jobs:
             OUT_FILE_NAME: deploy-freva.exe
             ASSET_MIME: application/vnd.microsoft.portable-executable
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python 3.11
         uses: actions/setup-python@v4
         with:

.github/workflows/ci_job.yml

@@ -11,7 +11,7 @@ jobs:
     steps:
     -
       name: Checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
     -
       name: Set up Python 3.11
       uses: actions/setup-python@v4
@@ -34,7 +34,7 @@ jobs:
     steps:
     -
       name: Checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
     -
       name: Set up Python
       uses: actions/setup-python@v4

.github/workflows/docker_job.yml

@@ -16,7 +16,7 @@ jobs:
     steps:
       -
         name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 

README.md

@@ -9,9 +9,9 @@ environments. The general strategy is to split the deployment into
 - Deploy a Hashicorp Vault service for storing and retrieving passwords
   and other sensitive data via docker
   (this step get automatically activated once the MariaDB service is set)
-- Deploy Apache Solr service via docker
+- Deploy [Databrowser API](https://github.com/FREVA-CLINT/databrowserAPI) service via docker
 - Deploy command line interface backend ([evaluation_system](https://github.com/FREVA-CLINT/freva))
-- Deploy web front end ([freva_web](https://gitlab.dkrz.de/freva/freva_web))
+- Deploy web front end ([freva_web](https://github.com/FREVA-CLINT/freva-web))
 
 
 > **_Note:_** A vault server is auto deployed once the mariadb server is deployed.
@@ -96,13 +96,15 @@ export PATH=$PATH:$HOME/.local/bin
 ```
 
 
-## Installing docker/podman and sudo access to the service servers
-Since the services of MariaDB, Apache Solr and Apache web will be deployed on
-docker container images, docker needs to be available on the target servers.
-Usually installing and running docker requires *root* privileges.
-Hence, on the servers that will be running docker you will need root access.
-There exist an option to install and run docker without root,
-information on a root-less docker option
+## Installing docker-compose/podman-compose and sudo access to the service servers
+Because the services of MariaDB, DatabrowserAPI and Apache httpd will be deployed
+on docker container images, docker needs to be available on the target servers.
+Since version *v2309.0.0* of the deployment the containers are set up
+using `doker-compose`. Hence `docker-compose` (or `podman-compose`) has to be
+installed on the host systems. Usually installing and running docker
+requires *root* privileges. Hence, on the servers that will be running docker
+you will need root access. There exists an option to install and run docker
+without root, information on a root-less docker option
 can be found [on the docker docs](https://docs.docker.com/engine/security/rootless/)
 > **_Note:_** Some systems use `podman` instead of `docker`. The deployment
 routine is able to distinguish and use the right service.

assets/share/freva/deployment/config/inventory.toml

@@ -49,8 +49,8 @@ hosts = ""
 ## leading 0s:
 
 ## hosts = "db-[99:101]-node.example.com"
-[solr]
-## Set the hostnames running the apache solr service
+[databrowser]
+## Set the hostnames running the databrowser service
 hosts = ""
 
 [db]
@@ -103,12 +103,15 @@ wipe = false
 db_playbook = ""
 vault_playbook = ""
 
-[solr.config]
+[databrowser.config]
 ## Config variables for the solr service
-port = 8983
+solr_port = 8983
 
 # Set the memory for the solr server
-mem = "4g"
+solr_mem = "4g"
+
+# Set the port the databrowser should be running on
+databrowser_port = 7777
 
 ## Set the become (sudo) user name to change to for installing the services
 ansible_become_user = "root"
@@ -128,7 +131,7 @@ wipe = false
 ## located in the user config directory.
 ## NOTE: only adjust this if you know what you are doing, if you leave this
 ## blank the system will used the default playbook (standard procedure)
-solr_playbook = ""
+databrowser_playbook = ""
 
 
 
@@ -316,5 +319,3 @@ web_playbook = ""
 
 ## Set the become (sudo) user name to change to for installing the services
 ansible_become_user = "root"
-
-

assets/share/freva/deployment/playbooks/databrowser-server-compose-template.yml

@@ -0,0 +1,61 @@
+version: '3'
+services:
+  {{solr_name}}:
+    image: solr:latest
+    container_name: {{solr_name}}
+    hostname: {{solr_name}}
+    networks:
+      - {{databrowser_name}}
+    environment:
+      - CORE=files
+      - NUM_BACKUPS=7
+      - SOLR_HEAP={{databrowser_solr_mem}}
+    volumes:
+{% for volume in solr_volumes %}
+      - {{volume}}
+{% endfor %}
+    tty: true
+    ports:
+      - {{databrowser_solr_port}}:8983
+
+  {{mongo_name}}:
+    image: mongo:latest
+    container_name: {{mongo_name}}
+    hostname: {{mongo_name}}
+    networks:
+      - {{databrowser_name}}
+    environment:
+      - MONGO_INITDB_ROOT_USERNAME=mongo
+      - MONGO_INITDB_ROOT_PASSWORD={{ root_passwd}}
+      - MONGO_INITDB_DATABASE=search_stats
+    volumes:
+{% for volume in mongo_volumes %}
+      - {{volume}}
+{% endfor %}
+    ports:
+      - 27017:27017
+    tty: true
+
+  {{databrowser_name}}:
+    image: ghcr.io/freva-clint/databrowserapi:latest
+    hostname: {{databrowser_name}}
+    networks:
+      - {{databrowser_name}}
+    environment:
+      - SOLR_CORE=files
+      - SOLR_HOST={{solr_name}}:8983
+      - MONGO_HOST={{mongo_name}}:27017
+      - MONGO_PASSWORD={{ root_passwd}}
+      - MONGO_DB=search_stats
+      - MONGO_USER=mongo
+    ports:
+      - {{databrowser_port}}:8080
+    container_name: {{databrowser_name}}
+    depends_on:
+      - {{solr_name}}
+      - {{mongo_name}}
+    tty: true
+
+networks:
+  {{databrowser_name}}:
+    driver: bridge

assets/share/freva/deployment/playbooks/databrowser-server-playbook.yml

@@ -0,0 +1,151 @@
+---
+- hosts: databrowser
+
+  vars:
+    ansible_python_interpreter: "{{ databrowser_ansible_python_interpreter }}"
+    databrowser_name: "{{project_name}}-databrowser"
+    compose_file: /opt/freva/compose_services/{{databrowser_name}}-compose.yml
+    solr_name: "{{project_name}}-solr"
+    mongo_name: "{{project_name}}-mongo"
+    solr_volumes:
+      - /opt/freva/{{project_name}}/solr_service:/var/solr/data:z
+      - /opt/freva/freva-service-config/solr/managed-schema.xml:/opt/solr/managed-schema.xml:z
+      - /opt/freva/freva-service-config/solr/create_cores.sh:/docker-entrypoint-initdb.d/create_cores.sh:z
+      - /opt/freva/freva-service-config/solr/synonyms.txt:/opt/solr/synonyms.txt:z
+      - /opt/freva/freva-service-config/solr/daily_backup.sh:/usr/local/bin/daily_backup:z
+    mongo_volumes:
+      - /opt/freva/{{project_name}}/databrowser/stats:/data/db:z
+    ansible_become_user: "{{ databrowser_ansible_become_user | default('root') }}"
+    use_become: "{{ databrowser_ansible_become_user is defined and databrowser_ansible_become_user != '' }}"
+  tasks:
+    - name: Copying docker/podman wrapper script
+      copy:
+        src: "{{ asset_dir }}/scripts/docker-or-podman"
+        dest: /tmp/docker-or-podman
+        mode: "0775"
+    - name: Registering systemctl path
+      stat:
+        path: /usr/bin/systemctl
+      register: systemctl
+    - name: Registering anacron path
+      stat:
+        path: /etc/cron.daily
+      register: cron
+    - name: Stopping services and deleting existing containers
+      become: "{{use_become}}"
+      shell: >
+        systemctl stop {{solr_name}} {{mongo_name}} {{databrowser_name}};
+        systemctl disable {{solr_name}} {{mongo_name}} {{databrowser_name}};
+        systemctl reset-failed;
+        /tmp/docker-or-podman rm {{solr_name}} {{mongo_name}} {{databrowser_name}};
+        /tmp/docker-or-podman rmi -f {{solr_name}} {{mongo_name}} {{databrowser_name}};
+        /tmp/docker-or-podman rmi -f solr mongo ghcr.io/freva-clint/databrowserapi;
+        exit 0
+      ignore_errors: true
+    - pause: seconds=5
+    - name: Deleting old systemd structure
+      file:
+        state: absent
+        path: "/etc/systemd/system/{{item}}.service"
+        force: true
+      with_items:
+        - "{{solr_name}}"
+        - "{{mongo_name}}"
+        - "{{databrowser_name}}"
+      become: "{{use_become}}"
+      when: systemctl.stat.exists == true
+    - name: Deleting config file
+      file:
+        state: absent
+        force: true
+        path: /opt/freva/freva-service-config/
+      become: "{{use_become}}"
+    - name: Cleaning existing directory structure
+      file:
+        path: "{item}"
+        state: absent
+      with_items:
+        - "/opt/freva/{{ project_name }}/solr_service"
+        - "/opt/freva/{{ project_name }}/databrowser"
+      when: databrowser_wipe == true
+      become: "{{use_become}}"
+    - name: Creating solr directory structure
+      file:
+        path: /opt/freva/{{ project_name }}/solr_service
+        state: directory
+        owner: 8983
+        group: 8983
+        recurse: true
+      become: "{{use_become}}"
+    - name: Creating compose directory structure
+      file:
+        path: /opt/freva/compose_services
+        state: directory
+        recurse: true
+      become: "{{use_become}}"
+    - name: Creating directory structure
+      file:
+        path: /opt/freva/{{ project_name }}/databrowser
+        state: directory
+        recurse: true
+      become: "{{use_become}}"
+    - name: Getting additional configurations
+      git:
+        repo: https://github.com/FREVA-CLINT/freva-service-config.git
+        dest: /opt/freva/freva-service-config
+        update: true
+      become: "{{use_become}}"
+    - name: Copy systemd files
+      copy:
+        src: "{{ asset_dir }}/scripts/create_systemd.py"
+        dest: /tmp/create_systemd.py
+        mode: "0755"
+      become: "{{use_become}}"
+    - name: Pulling container
+      shell:
+        cmd: /tmp/docker-or-podman pull {{item}}
+      become: "{{use_become}}"
+      with_items:
+        - "docker.io/solr:latest"
+        - "docker.io/mongo:latest"
+        - "ghcr.io/freva-clint/databrowserapi:latest"
+    - name: Copy cron create script to target machine
+      copy: src="{{ asset_dir }}/scripts/create_cron.sh" dest=/tmp/solr_service/
+      become: "{{use_become}}"
+    - name: Creating compose file
+      become: "{{use_become}}"
+      template:
+        src: "{{ asset_dir }}/playbooks/databrowser-server-compose-template.yml"
+        dest: "{{compose_file}}"
+    - name: Creating systemd services
+      shell: |
+        /tmp/create_systemd.py {{databrowser_name}} compose --enable --project-name {{databrowser_name}} -f {{compose_file}} up --remove-orphans
+      when: systemctl.stat.exists == true
+      become: "{{use_become}}"
+    - pause: seconds=3
+    - name: Creating cron jobs
+      shell: >
+        sh /tmp/solr_service/create_cron.sh "{{ solr_name }}" "{{databrowser_email}}"
+      when: systemctl.stat.exists == true
+      become: "{{use_become}}"
+    - name: Deleting tmporary files
+      file:
+        path: "{{item}}"
+        state: absent
+      with_items:
+        - /tmp/docker-or-podman
+        - /tmp/create_systemd.py
+        - /tmp/solr_service
+      become: "{{use_become}}"
+- hosts: web
+  vars:
+    use_become: "{{ web_ansible_become_user is defined and web_ansible_become_user != '' }}"
+  tasks:
+    - name: Registering systemctl path
+      stat:
+        path: /usr/bin/systemctl
+      register: systemctl
+    - name: Restarting web container
+      shell: systemctl restart {{web_name}}; echo 0
+      become: "{{use_become}}"
+      when: systemctl.stat.exists == true

assets/share/freva/deployment/playbooks/db-server-compose-template.yml

@@ -0,0 +1,29 @@
+version: '3'
+services:
+  {{db_name}}:
+    image: mariadb:latest
+    hostname: {{db_name}}
+    networks:
+      - {{db_name}}
+    environment:
+      - ROOT_PW={{ root_passwd }}
+      - HOST={{ db_host }}
+      - NUM_BACKUPS=7
+      - PROJECT={{ project_name }}
+      - MYSQL_USER={{db_user}}
+      - MYSQL_PASSWORD={{db_passwd}}
+      - MYSQL_DATABASE={{db}}
+      - MYSQL_ROOT_PASSWORD={{ root_passwd }}
+      - BACKUP_DIR=/var/lib/mysql/backup
+    volumes:
+{% for volume in db_volumes %}
+      - {{volume}}
+{% endfor %}
+    container_name: {{db_name}}
+    tty: true
+    ports:
+      - {{db_port}}:3306
+
+networks:
+  {{db_name}}:
+    driver: bridge