remote: added toggles for mount and union

moved settings to its own key to allow us to keep the sorting and not worry about remote being first.

defaults/settings.yml.default

@@ -4,16 +4,19 @@ authelia:
   subdomain: login
 downloads: /mnt/unionfs/downloads
 rclone:
-  enabled: true
+  enabled: yes
   remotes:
     - remote: google
-      template: google
-      upload: true
-      upload_from: /mnt/local/Media
-      vfs_cache:
-        enabled: false
-        max_age: 504h
-        size: 50G
+      settings:
+        mount: yes
+        template: google
+        union: yes
+        upload: yes
+        upload_from: /mnt/local/Media
+        vfs_cache:
+          enabled: no
+          max_age: 504h
+          size: 50G
   version: latest
 shell: bash
 transcodes: /mnt/local/transcodes

roles/cloudplow/tasks/main.yml

@@ -23,13 +23,13 @@
   ansible.builtin.set_fact:
     cloudplow_remotes: "{{ cloudplow_remotes + [item] }}"
   loop: "{{ rclone.remotes }}"
-  when: item.upload
+  when: item.settings.upload
 
 - name: Filter Rclone upload_from on remotes
   ansible.builtin.set_fact:
-    cloudplow_upload_paths_unique: "{{ cloudplow_upload_paths_unique + [item.upload_from] }}"
+    cloudplow_upload_paths_unique: "{{ cloudplow_upload_paths_unique + [item.settings.upload_from] }}"
   loop: "{{ rclone.remotes }}"
-  when: item.upload and (item.upload_from not in cloudplow_upload_paths_unique)
+  when: item.settings.upload and (item.settings.upload_from not in cloudplow_upload_paths_unique)
 
 - name: Cloudplow tasks
   when: (cloudplow_remotes | length > 0)
@@ -58,8 +58,8 @@
     - name: Set Python version
       ansible.builtin.set_fact:
         cloudplow_python: "{{ 'python3'
-                          if ansible_distribution_version is version('20.04', '==')
-                          else 'python3.8' }}"
+                           if ansible_distribution_version is version('20.04', '==')
+                           else 'python3.8' }}"
 
     - name: "Execute Python role"
       ansible.builtin.include_role:

roles/cloudplow/templates/config.json.j2

@@ -41,7 +41,7 @@
         "*.db"
       ],
       "rclone_extras": {
-        {{ lookup('vars', 'cloudplow_rclone_' + remote.template + '_template', default=cloudplow_rclone_google_template) | indent(width=8, first=false) }}
+        {{ lookup('vars', 'cloudplow_rclone_' + remote.settings.template + '_template', default=cloudplow_rclone_google_template) | indent(width=8, first=false) }}
       },
       "rclone_sleeps": {
         "Failed to copy: googleapi: Error 403: User rate limit exceeded": {
@@ -58,13 +58,13 @@
       "remove_empty_dir_depth": 2,
       "sync_remote": "{{ (remote.remote if ':' in remote.remote else remote.remote
                           + ':' + lookup('vars', 'cloudplow_remote_' + (remote.remote.split(':')[0] if ':' in remote.remote else remote.remote) + '_folder', default=cloudplow_remote_default_folder))
-                          if (remote.template != 'nfs')
+                          if (remote.settings.template != 'nfs')
                           else ('/mnt/remote/' + remote.remote.split(':')[0] if ':' in remote.remote else remote.remote)
                           + lookup('vars', 'cloudplow_remote_' + (remote.remote.split(':')[0] if ':' in remote.remote else remote.remote) + '_folder', default=cloudplow_remote_default_folder) }}",
-      "upload_folder": "{{ remote.upload_from }}",
+      "upload_folder": "{{ remote.settings.upload_from }}",
       "upload_remote": "{{ (remote.remote if ':' in remote.remote else remote.remote
                             + ':' + lookup('vars', 'cloudplow_remote_' + (remote.remote.split(':')[0] if ':' in remote.remote else remote.remote) + '_folder', default=cloudplow_remote_default_folder))
-                            if (remote.template != 'nfs')
+                            if (remote.settings.template != 'nfs')
                             else ('/mnt/remote/' + remote.remote.split(':')[0] if ':' in remote.remote else remote.remote)
                             + lookup('vars', 'cloudplow_remote_' + (remote.remote.split(':')[0] if ':' in remote.remote else remote.remote) + '_folder', default=cloudplow_remote_default_folder) }}"
     {% if loop.index == loop.length %}}{% else %}},{{ '\n' }}{% endif %}

roles/docker/tasks/main.yml

@@ -29,7 +29,7 @@
   ansible.builtin.set_fact:
     _remotes_services_list: "{{ _remotes_services_list | default('') + (rclone_service_template + rclone_remote_name + '.service ') }}"
   with_items: "{{ rclone.remotes }}"
-  when: rclone_remote_is_defined and use_remote and item.template != "nfs"
+  when: rclone_remote_is_defined and use_remote and item.settings.template != "nfs" and item.settings.mount
 
 - name: Tasks for when Docker exists and is running
   when: docker_service_check

roles/remote/tasks/main.yml

@@ -167,9 +167,9 @@
     - name: Remote Tasks
       ansible.builtin.include_tasks: "remote.yml"
       with_items: "{{ rclone.remotes }}"
-      when: rclone_remote_is_defined and item.template != "nfs"
+      when: rclone_remote_is_defined and item.settings.template != "nfs" and item.settings.mount
 
     - name: Remote Tasks (NFS)
       ansible.builtin.include_tasks: "nfs.yml"
       with_items: "{{ rclone.remotes }}"
-      when: rclone_remote_is_defined and item.template == "nfs"
+      when: rclone_remote_is_defined and item.settings.template == "nfs" and item.settings.mount

roles/remote/tasks/remote.yml

@@ -83,7 +83,7 @@
 
 - name: Remote | Import '{{ _service_file }}'
   ansible.builtin.template:
-    src: "{{ item.template }}{{ '' if item.template is regex('^/.*') else '.j2' }}"
+    src: "{{ item.settings.template }}{{ '' if item.settings.template is regex('^/.*') else '.j2' }}"
     dest: "/etc/systemd/system/{{ _service_file }}"
     mode: "0644"
     force: true
@@ -101,31 +101,31 @@
     dest: "/etc/systemd/system/{{ _service_refresh }}.timer"
     force: true
     mode: "0644"
-  when: (item.template != "sftp")
+  when: (item.settings.template != "sftp")
 
 - name: Remote | Systemd daemon-reload '{{ _service_refresh }}.timer'
   ansible.builtin.systemd:
     name: "{{ _service_refresh }}.timer"
     state: stopped
     enabled: false
     daemon_reload: true
-  when: (item.template != "sftp")
+  when: (item.settings.template != "sftp")
 
 - name: Remote | Import '{{ _service_refresh }}.service'
   ansible.builtin.template:
     src: "rclone_refresh.service.j2"
     dest: "/etc/systemd/system/{{ _service_refresh }}.service"
     force: true
     mode: "0644"
-  when: (item.template != "sftp")
+  when: (item.settings.template != "sftp")
 
 - name: Remote | Systemd daemon-reload '{{ _service_refresh }}.service'
   ansible.builtin.systemd:
     name: "{{ _service_refresh }}.service"
     state: stopped
     enabled: false
     daemon_reload: true
-  when: (item.template != "sftp")
+  when: (item.settings.template != "sftp")
 
 - name: Remote | Get list of Rclone remotes in config
   ansible.builtin.shell: rclone listremotes
@@ -175,15 +175,15 @@
             name: "{{ _service_refresh }}.timer"
             state: started
             enabled: true
-          when: (item.template != "sftp")
+          when: (item.settings.template != "sftp")
 
         - name: Remote | Start '{{ _service_refresh }}.service'
           ansible.builtin.systemd:
             name: "{{ _service_refresh }}.service"
             state: started
             enabled: true
             no_block: true
-          when: (item.template != "sftp")
+          when: (item.settings.template != "sftp")
 
     - name: "Remote | Display error message when Rclone remote is not working properly"
       ansible.builtin.fail:

roles/remote/templates/dropbox.j2

@@ -22,7 +22,7 @@ ExecStart=/usr/bin/rclone mount \
 {% if mounts.ipv4_only %}
     --bind={{ ansible_default_ipv4.address }} \
 {% endif %}
-{% if (rclone_vfs_cache_dir_lookup | length > 0) and item.vfs_cache.enabled %}
+{% if (rclone_vfs_cache_dir_lookup | length > 0) and item.settings.vfs_cache.enabled %}
     --cache-dir={{ rclone_vfs_cache_dir_lookup }} \
 {% endif %}
     --config={{ rclone_config_path }} \
@@ -39,25 +39,25 @@ ExecStart=/usr/bin/rclone mount \
     --umask=002 \
     --use-mmap \
     --user-agent='{{ user_agent }}' \
-{% if item.vfs_cache.enabled %}
+{% if item.settings.vfs_cache.enabled %}
     --vfs-cache-min-free-space={{ rclone_vfs_cache_min_free_space }} \
-    --vfs-cache-max-age={{ item.vfs_cache.max_age | default('504h') }} \
-    --vfs-cache-max-size={{ item.vfs_cache.size | default('50G') }} \
+    --vfs-cache-max-age={{ item.settings.vfs_cache.max_age | default('504h') }} \
+    --vfs-cache-max-size={{ item.settings.vfs_cache.size | default('50G') }} \
     --vfs-cache-mode=full \
     --vfs-cache-poll-interval={{ rclone_vfs_cache_poll_interval }} \
     --vfs-fast-fingerprint \
     --vfs-read-ahead=128M \
 {% endif %}
     --vfs-read-chunk-size-limit=2G \
-    --vfs-read-chunk-size={{ '32M' if item.vfs_cache.enabled else '64M' }} \
+    --vfs-read-chunk-size={{ '32M' if item.settings.vfs_cache.enabled else '64M' }} \
     -v \
     "{{ rclone_remote_with_path }}" "/mnt/remote/{{ rclone_remote_name }}"
 ExecStop=/bin/fusermount3 -uz "/mnt/remote/{{ rclone_remote_name }}"
 Restart=on-abort
 RestartSec=5
 StartLimitInterval=60s
 StartLimitBurst=3
-{% if item.vfs_cache.enabled %}
+{% if item.settings.vfs_cache.enabled %}
 TimeoutSec=21600
 LimitNOFILE=infinity
 LimitMEMLOCK=infinity

roles/remote/templates/google.j2

@@ -22,13 +22,13 @@ ExecStart=/usr/bin/rclone mount \
 {% if mounts.ipv4_only %}
     --bind={{ ansible_default_ipv4.address }} \
 {% endif %}
-{% if (rclone_vfs_cache_dir_lookup | length > 0) and item.vfs_cache.enabled %}
+{% if (rclone_vfs_cache_dir_lookup | length > 0) and item.settings.vfs_cache.enabled %}
     --cache-dir={{ rclone_vfs_cache_dir_lookup }} \
 {% endif %}
     --config={{ rclone_config_path }} \
-    --buffer-size={{ '32M' if item.vfs_cache.enabled else '64M' }} \
+    --buffer-size={{ '32M' if item.settings.vfs_cache.enabled else '64M' }} \
     --dir-cache-time={{ rclone_cloud_dir_cache_time }} \
-{% if not item.vfs_cache.enabled %}
+{% if not item.settings.vfs_cache.enabled %}
     --drive-chunk-size=64M \
 {% endif %}
     --drive-pacer-burst=1000 \
@@ -43,25 +43,25 @@ ExecStart=/usr/bin/rclone mount \
     --umask=002 \
     --use-mmap \
     --user-agent='{{ user_agent }}' \
-{% if item.vfs_cache.enabled %}
+{% if item.settings.vfs_cache.enabled %}
     --vfs-cache-min-free-space={{ rclone_vfs_cache_min_free_space }} \
-    --vfs-cache-max-age={{ item.vfs_cache.max_age | default('504h') }} \
-    --vfs-cache-max-size={{ item.vfs_cache.size | default('50G') }} \
+    --vfs-cache-max-age={{ item.settings.vfs_cache.max_age | default('504h') }} \
+    --vfs-cache-max-size={{ item.settings.vfs_cache.size | default('50G') }} \
     --vfs-cache-mode=full \
     --vfs-cache-poll-interval={{ rclone_vfs_cache_poll_interval }} \
     --vfs-fast-fingerprint \
     --vfs-read-ahead=128M \
 {% endif %}
     --vfs-read-chunk-size-limit=2G \
-    --vfs-read-chunk-size={{ '32M' if item.vfs_cache.enabled else '64M' }} \
+    --vfs-read-chunk-size={{ '32M' if item.settings.vfs_cache.enabled else '64M' }} \
     -v \
     "{{ rclone_remote_with_path }}" "/mnt/remote/{{ rclone_remote_name }}"
 ExecStop=/bin/fusermount3 -uz "/mnt/remote/{{ rclone_remote_name }}"
 Restart=on-abort
 RestartSec=5
 StartLimitInterval=60s
 StartLimitBurst=3
-{% if item.vfs_cache.enabled %}
+{% if item.settings.vfs_cache.enabled %}
 TimeoutSec=21600
 LimitNOFILE=infinity
 LimitMEMLOCK=infinity

roles/remote/templates/sftp.j2

@@ -19,7 +19,7 @@ Type=notify
 ExecStartPre=/bin/sleep 10
 ExecStart=/usr/bin/rclone mount \
     --allow-other \
-{% if (rclone_vfs_cache_dir_lookup | length > 0) and item.vfs_cache.enabled %}
+{% if (rclone_vfs_cache_dir_lookup | length > 0) and item.settings.vfs_cache.enabled %}
     --cache-dir={{ rclone_vfs_cache_dir_lookup }} \
 {% endif %}
     --config={{ rclone_config_path }} \
@@ -35,10 +35,10 @@ ExecStart=/usr/bin/rclone mount \
     --syslog \
     --umask=002 \
     --user-agent='{{ user_agent }}' \
-{% if item.vfs_cache.enabled %}
+{% if item.settings.vfs_cache.enabled %}
     --vfs-cache-min-free-space={{ rclone_vfs_cache_min_free_space }} \
-    --vfs-cache-max-age={{ item.vfs_cache.max_age | default('504h') }} \
-    --vfs-cache-max-size={{ item.vfs_cache.size | default('50G') }} \
+    --vfs-cache-max-age={{ item.settings.vfs_cache.max_age | default('504h') }} \
+    --vfs-cache-max-size={{ item.settings.vfs_cache.size | default('50G') }} \
     --vfs-cache-mode=full \
     --vfs-cache-poll-interval={{ rclone_vfs_cache_poll_interval }} \
     --vfs-fast-fingerprint \

roles/settings/tasks/subtasks/migrator/settings_yml/migration_01.yml

@@ -38,21 +38,21 @@
 
 - name: Migrator | 'settings.yml' | Migration 01 | Create 'rclone.remotes' dict
   ansible.builtin.shell: |
-    yyq e -i '.rclone.remotes += [{"remote": "{{ rclone.remote }}", "template": "google", "upload": true, "upload_from": "/mnt/local/Media", "vfs_cache": {"enabled": false, "size": "50G", "max_age": "504h"}}]' {{ playbook_dir }}/{{ file }}
+    yyq e -i '.rclone.remotes += [{"remote": "{{ rclone.remote }}", "settings": {"mounts": true, "template": "google", "union": true, "upload": true, "upload_from": "/mnt/local/Media", "vfs_cache": {"enabled": false, "size": "50G", "max_age": "504h"}}}]' {{ playbook_dir }}/{{ file }}
   become: true
   become_user: "{{ saltbox_yml.stat.pw_name }}"
   when: (rclone.remotes is undefined) and rclone_old_remote_is_defined
 
 - name: Migrator | 'settings.yml' | Migration 01 | Create 'rclone.remotes' dict
   ansible.builtin.shell: |
-    yyq e -i '.rclone.remotes += [{"remote": "google", "template": "google", "upload": true, "upload_from": "/mnt/local/Media", "vfs_cache": {"enabled": false, "size": "50G", "max_age": "504h"}}]' {{ playbook_dir }}/{{ file }}
+    yyq e -i '.rclone.remotes += [{"remote": "google", "settings": {"mounts": true, "template": "google", "union": true, "upload": true, "upload_from": "/mnt/local/Media", "vfs_cache": {"enabled": false, "size": "50G", "max_age": "504h"}}}]' {{ playbook_dir }}/{{ file }}
   become: true
   become_user: "{{ saltbox_yml.stat.pw_name }}"
   when: (rclone.remotes is undefined) and not rclone_old_remote_is_defined
 
 - name: Migrator | 'settings.yml' | Migration 01 | Add feeder to remotes
   ansible.builtin.shell: |
-    yyq e -i '.rclone.remotes += [{"remote": "feeder", "template": "sftp", "upload": false, "upload_from": "/mnt/local/Media", "vfs_cache": {"enabled": false, "size": "50G", "max_age": "504h"}}]' {{ playbook_dir }}/{{ file }}
+    yyq e -i '.rclone.remotes += [{"remote": "feeder", "settings": {"mounts": true, "template": "sftp", "union": true, "upload": false, "upload_from": "/mnt/local/Media", "vfs_cache": {"enabled": false, "size": "50G", "max_age": "504h"}}}]' {{ playbook_dir }}/{{ file }}
   become: true
   become_user: "{{ saltbox_yml.stat.pw_name }}"
   when: old_feeder_mount
@@ -64,6 +64,50 @@
   become_user: "{{ saltbox_yml.stat.pw_name }}"
   when: (rclone.enabled is undefined)
 
+- name: Migrator | 'settings.yml' | Migration 01 | Upgrade to new settings format
+  when: (rclone.remotes is defined)
+  block:
+    - name: Migrator | 'settings.yml' | Migration 01 | Read current config file
+      ansible.builtin.slurp:
+        src: /srv/git/saltbox/settings.yml
+      register: settings_config_content
+
+    - name: Migrator | 'settings.yml' | Migration 01 | Parse the configuration
+      ansible.builtin.set_fact:
+        settings_config: "{{ settings_config_content['content'] | b64decode | from_yaml }}"
+
+    - name: Migrator | 'settings.yml' | Migration 01 | Transform the data structure
+      ansible.builtin.set_fact:
+        new_remotes: "{{ new_remotes | default([]) + [{'remote': item.remote, 'settings': {'mount': true, 'template': item.template, 'union': true, 'upload': item.upload, 'upload_from': item.upload_from, 'vfs_cache': {'enabled': item.vfs_cache.enabled, 'max_age': item.vfs_cache.max_age, 'size': item.vfs_cache.size}}}] }}"
+      loop: "{{ settings_config.rclone.remotes }}"
+      when: item.template is defined
+
+    - name: Migrator | 'settings.yml' | Migration 01 | Combine new structure with original config
+      ansible.builtin.set_fact:
+        new_config: "{{ (settings_config | combine({'rclone': {'remotes': new_remotes}}, recursive=True)) }}"
+      when: new_remotes is defined
+
+    - name: Migrator | 'settings.yml' | Migration 01 | Write the new configuration file
+      ansible.builtin.copy:
+        dest: /srv/git/saltbox/settings.yml
+        content: "{{ new_config | to_nice_yaml }}"
+        owner: "{{ saltbox_yml.stat.uid }}"
+        group: "{{ saltbox_yml.stat.gid }}"
+        mode: "0664"
+      when: new_remotes is defined
+
+- name: Migrator | 'settings.yml' | Migration 01 | Convert 'true' to 'yes'
+  ansible.builtin.command: >
+    yyq eval '
+      (.. | select(tag == "!!bool" and . == true)) |= "yes"
+    ' /srv/git/saltbox/settings.yml -i
+
+- name: Migrator | 'settings.yml' | Migration 01 | Convert 'false' to 'no'
+  ansible.builtin.command: >
+    yyq eval '
+      (.. | select(tag == "!!bool" and . == false)) |= "no"
+    ' /srv/git/saltbox/settings.yml -i
+
 - name: Migrator | 'settings.yml' | Migration 01 | Remove 'null' values
   ansible.builtin.replace:
     path: "{{ playbook_dir }}/{{ file }}"

roles/unionfs/defaults/main.yml

@@ -49,6 +49,9 @@ mergerfs_download_backup_url: "
 
 mergerfs_mount_branches: "{{ local_mount_branch }}{{ _remotes_list }}"
 
+mergerfs_branch_mode: "NC"
+mergerfs_remote_branch_mode_lookup: "{{ lookup('vars', 'mergerfs_remote_' + rclone_remote_name + '_branch_mode', default=mergerfs_branch_mode) }}"
+
 mergerfs_mount_service_after: "network-online.target"
 
 mergerfs_mount_readdir_policy: "seq"

roles/unionfs/tasks/subtasks/docker/daemon.yml

@@ -11,7 +11,7 @@
   ansible.builtin.set_fact:
     _remotes_services_list: "{{ _remotes_services_list | default('') + (rclone_service_template + rclone_remote_name + '.service ') }}"
   with_items: "{{ rclone.remotes }}"
-  when: rclone_remote_is_defined and use_remote and item.template != "nfs"
+  when: rclone_remote_is_defined and use_remote and item.settings.template != "nfs" and item.settings.mount
 
 - name: "Docker | Daemon | Create override directory"
   ansible.builtin.file:

roles/unionfs/tasks/subtasks/variables.yml

@@ -12,9 +12,9 @@
 ################################
 - name: Variables | Set '_remotes_list' variable
   ansible.builtin.set_fact:
-    _remotes_list: "{{ _remotes_list | default('') + '/mnt/remote/' + (item.remote.split(':')[0] if ':' in item.remote else item.remote) + '=NC:' }}"
+    _remotes_list: "{{ _remotes_list | default('') + '/mnt/remote/' + (item.remote.split(':')[0] if ':' in item.remote else item.remote) + '=' + mergerfs_remote_branch_mode_lookup + ':' }}"
   with_items: "{{ rclone.remotes }}"
-  when: rclone_remote_is_defined and use_remote
+  when: rclone_remote_is_defined and use_remote and item.settings.mount and item.settings.union
 
 - name: Variables | Append 'custom_mount_branch' to '_remotes_list' variable
   ansible.builtin.set_fact: