From d0b3be81597e2f000107c4966a56fd4e972f596c Mon Sep 17 00:00:00 2001 From: saltydk Date: Tue, 21 Nov 2023 13:15:51 +0100 Subject: [PATCH] remote: added toggles for mount and union moved settings to its own key to allow us to keep the sorting and not worry about remote being first. --- defaults/settings.yml.default | 19 ++++--- roles/cloudplow/tasks/main.yml | 10 ++-- roles/cloudplow/templates/config.json.j2 | 8 +-- roles/docker/tasks/main.yml | 2 +- roles/remote/tasks/main.yml | 4 +- roles/remote/tasks/remote.yml | 14 +++--- roles/remote/templates/dropbox.j2 | 12 ++--- roles/remote/templates/google.j2 | 16 +++--- roles/remote/templates/sftp.j2 | 8 +-- .../migrator/settings_yml/migration_01.yml | 50 +++++++++++++++++-- roles/unionfs/defaults/main.yml | 3 ++ .../unionfs/tasks/subtasks/docker/daemon.yml | 2 +- roles/unionfs/tasks/subtasks/variables.yml | 4 +- 13 files changed, 101 insertions(+), 51 deletions(-) diff --git a/defaults/settings.yml.default b/defaults/settings.yml.default index 3d8bc2d561..5e809e65df 100644 --- a/defaults/settings.yml.default +++ b/defaults/settings.yml.default @@ -4,16 +4,19 @@ authelia: subdomain: login downloads: /mnt/unionfs/downloads rclone: - enabled: true + enabled: yes remotes: - remote: google - template: google - upload: true - upload_from: /mnt/local/Media - vfs_cache: - enabled: false - max_age: 504h - size: 50G + settings: + mount: yes + template: google + union: yes + upload: yes + upload_from: /mnt/local/Media + vfs_cache: + enabled: no + max_age: 504h + size: 50G version: latest shell: bash transcodes: /mnt/local/transcodes diff --git a/roles/cloudplow/tasks/main.yml b/roles/cloudplow/tasks/main.yml index 1e277a4c5e..3f2dec6564 100644 --- a/roles/cloudplow/tasks/main.yml +++ b/roles/cloudplow/tasks/main.yml @@ -23,13 +23,13 @@ ansible.builtin.set_fact: cloudplow_remotes: "{{ cloudplow_remotes + [item] }}" loop: "{{ rclone.remotes }}" - when: item.upload + when: item.settings.upload - name: Filter Rclone upload_from on remotes ansible.builtin.set_fact: - cloudplow_upload_paths_unique: "{{ cloudplow_upload_paths_unique + [item.upload_from] }}" + cloudplow_upload_paths_unique: "{{ cloudplow_upload_paths_unique + [item.settings.upload_from] }}" loop: "{{ rclone.remotes }}" - when: item.upload and (item.upload_from not in cloudplow_upload_paths_unique) + when: item.settings.upload and (item.settings.upload_from not in cloudplow_upload_paths_unique) - name: Cloudplow tasks when: (cloudplow_remotes | length > 0) @@ -58,8 +58,8 @@ - name: Set Python version ansible.builtin.set_fact: cloudplow_python: "{{ 'python3' - if ansible_distribution_version is version('20.04', '==') - else 'python3.8' }}" + if ansible_distribution_version is version('20.04', '==') + else 'python3.8' }}" - name: "Execute Python role" ansible.builtin.include_role: diff --git a/roles/cloudplow/templates/config.json.j2 b/roles/cloudplow/templates/config.json.j2 index 11bac847d4..3be0016177 100644 --- a/roles/cloudplow/templates/config.json.j2 +++ b/roles/cloudplow/templates/config.json.j2 @@ -41,7 +41,7 @@ "*.db" ], "rclone_extras": { - {{ lookup('vars', 'cloudplow_rclone_' + remote.template + '_template', default=cloudplow_rclone_google_template) | indent(width=8, first=false) }} + {{ lookup('vars', 'cloudplow_rclone_' + remote.settings.template + '_template', default=cloudplow_rclone_google_template) | indent(width=8, first=false) }} }, "rclone_sleeps": { "Failed to copy: googleapi: Error 403: User rate limit exceeded": { @@ -58,13 +58,13 @@ "remove_empty_dir_depth": 2, "sync_remote": "{{ (remote.remote if ':' in remote.remote else remote.remote + ':' + lookup('vars', 'cloudplow_remote_' + (remote.remote.split(':')[0] if ':' in remote.remote else remote.remote) + '_folder', default=cloudplow_remote_default_folder)) - if (remote.template != 'nfs') + if (remote.settings.template != 'nfs') else ('/mnt/remote/' + remote.remote.split(':')[0] if ':' in remote.remote else remote.remote) + lookup('vars', 'cloudplow_remote_' + (remote.remote.split(':')[0] if ':' in remote.remote else remote.remote) + '_folder', default=cloudplow_remote_default_folder) }}", - "upload_folder": "{{ remote.upload_from }}", + "upload_folder": "{{ remote.settings.upload_from }}", "upload_remote": "{{ (remote.remote if ':' in remote.remote else remote.remote + ':' + lookup('vars', 'cloudplow_remote_' + (remote.remote.split(':')[0] if ':' in remote.remote else remote.remote) + '_folder', default=cloudplow_remote_default_folder)) - if (remote.template != 'nfs') + if (remote.settings.template != 'nfs') else ('/mnt/remote/' + remote.remote.split(':')[0] if ':' in remote.remote else remote.remote) + lookup('vars', 'cloudplow_remote_' + (remote.remote.split(':')[0] if ':' in remote.remote else remote.remote) + '_folder', default=cloudplow_remote_default_folder) }}" {% if loop.index == loop.length %}}{% else %}},{{ '\n' }}{% endif %} diff --git a/roles/docker/tasks/main.yml b/roles/docker/tasks/main.yml index 501c1adaf4..17adc8f0da 100644 --- a/roles/docker/tasks/main.yml +++ b/roles/docker/tasks/main.yml @@ -29,7 +29,7 @@ ansible.builtin.set_fact: _remotes_services_list: "{{ _remotes_services_list | default('') + (rclone_service_template + rclone_remote_name + '.service ') }}" with_items: "{{ rclone.remotes }}" - when: rclone_remote_is_defined and use_remote and item.template != "nfs" + when: rclone_remote_is_defined and use_remote and item.settings.template != "nfs" and item.settings.mount - name: Tasks for when Docker exists and is running when: docker_service_check diff --git a/roles/remote/tasks/main.yml b/roles/remote/tasks/main.yml index 570bef496e..923e973acb 100644 --- a/roles/remote/tasks/main.yml +++ b/roles/remote/tasks/main.yml @@ -167,9 +167,9 @@ - name: Remote Tasks ansible.builtin.include_tasks: "remote.yml" with_items: "{{ rclone.remotes }}" - when: rclone_remote_is_defined and item.template != "nfs" + when: rclone_remote_is_defined and item.settings.template != "nfs" and item.settings.mount - name: Remote Tasks (NFS) ansible.builtin.include_tasks: "nfs.yml" with_items: "{{ rclone.remotes }}" - when: rclone_remote_is_defined and item.template == "nfs" + when: rclone_remote_is_defined and item.settings.template == "nfs" and item.settings.mount diff --git a/roles/remote/tasks/remote.yml b/roles/remote/tasks/remote.yml index 6548fa712e..d877578167 100644 --- a/roles/remote/tasks/remote.yml +++ b/roles/remote/tasks/remote.yml @@ -83,7 +83,7 @@ - name: Remote | Import '{{ _service_file }}' ansible.builtin.template: - src: "{{ item.template }}{{ '' if item.template is regex('^/.*') else '.j2' }}" + src: "{{ item.settings.template }}{{ '' if item.settings.template is regex('^/.*') else '.j2' }}" dest: "/etc/systemd/system/{{ _service_file }}" mode: "0644" force: true @@ -101,7 +101,7 @@ dest: "/etc/systemd/system/{{ _service_refresh }}.timer" force: true mode: "0644" - when: (item.template != "sftp") + when: (item.settings.template != "sftp") - name: Remote | Systemd daemon-reload '{{ _service_refresh }}.timer' ansible.builtin.systemd: @@ -109,7 +109,7 @@ state: stopped enabled: false daemon_reload: true - when: (item.template != "sftp") + when: (item.settings.template != "sftp") - name: Remote | Import '{{ _service_refresh }}.service' ansible.builtin.template: @@ -117,7 +117,7 @@ dest: "/etc/systemd/system/{{ _service_refresh }}.service" force: true mode: "0644" - when: (item.template != "sftp") + when: (item.settings.template != "sftp") - name: Remote | Systemd daemon-reload '{{ _service_refresh }}.service' ansible.builtin.systemd: @@ -125,7 +125,7 @@ state: stopped enabled: false daemon_reload: true - when: (item.template != "sftp") + when: (item.settings.template != "sftp") - name: Remote | Get list of Rclone remotes in config ansible.builtin.shell: rclone listremotes @@ -175,7 +175,7 @@ name: "{{ _service_refresh }}.timer" state: started enabled: true - when: (item.template != "sftp") + when: (item.settings.template != "sftp") - name: Remote | Start '{{ _service_refresh }}.service' ansible.builtin.systemd: @@ -183,7 +183,7 @@ state: started enabled: true no_block: true - when: (item.template != "sftp") + when: (item.settings.template != "sftp") - name: "Remote | Display error message when Rclone remote is not working properly" ansible.builtin.fail: diff --git a/roles/remote/templates/dropbox.j2 b/roles/remote/templates/dropbox.j2 index 3ecca9626c..769bf5668b 100644 --- a/roles/remote/templates/dropbox.j2 +++ b/roles/remote/templates/dropbox.j2 @@ -22,7 +22,7 @@ ExecStart=/usr/bin/rclone mount \ {% if mounts.ipv4_only %} --bind={{ ansible_default_ipv4.address }} \ {% endif %} -{% if (rclone_vfs_cache_dir_lookup | length > 0) and item.vfs_cache.enabled %} +{% if (rclone_vfs_cache_dir_lookup | length > 0) and item.settings.vfs_cache.enabled %} --cache-dir={{ rclone_vfs_cache_dir_lookup }} \ {% endif %} --config={{ rclone_config_path }} \ @@ -39,17 +39,17 @@ ExecStart=/usr/bin/rclone mount \ --umask=002 \ --use-mmap \ --user-agent='{{ user_agent }}' \ -{% if item.vfs_cache.enabled %} +{% if item.settings.vfs_cache.enabled %} --vfs-cache-min-free-space={{ rclone_vfs_cache_min_free_space }} \ - --vfs-cache-max-age={{ item.vfs_cache.max_age | default('504h') }} \ - --vfs-cache-max-size={{ item.vfs_cache.size | default('50G') }} \ + --vfs-cache-max-age={{ item.settings.vfs_cache.max_age | default('504h') }} \ + --vfs-cache-max-size={{ item.settings.vfs_cache.size | default('50G') }} \ --vfs-cache-mode=full \ --vfs-cache-poll-interval={{ rclone_vfs_cache_poll_interval }} \ --vfs-fast-fingerprint \ --vfs-read-ahead=128M \ {% endif %} --vfs-read-chunk-size-limit=2G \ - --vfs-read-chunk-size={{ '32M' if item.vfs_cache.enabled else '64M' }} \ + --vfs-read-chunk-size={{ '32M' if item.settings.vfs_cache.enabled else '64M' }} \ -v \ "{{ rclone_remote_with_path }}" "/mnt/remote/{{ rclone_remote_name }}" ExecStop=/bin/fusermount3 -uz "/mnt/remote/{{ rclone_remote_name }}" @@ -57,7 +57,7 @@ Restart=on-abort RestartSec=5 StartLimitInterval=60s StartLimitBurst=3 -{% if item.vfs_cache.enabled %} +{% if item.settings.vfs_cache.enabled %} TimeoutSec=21600 LimitNOFILE=infinity LimitMEMLOCK=infinity diff --git a/roles/remote/templates/google.j2 b/roles/remote/templates/google.j2 index bdf01549ca..d61a659cfa 100644 --- a/roles/remote/templates/google.j2 +++ b/roles/remote/templates/google.j2 @@ -22,13 +22,13 @@ ExecStart=/usr/bin/rclone mount \ {% if mounts.ipv4_only %} --bind={{ ansible_default_ipv4.address }} \ {% endif %} -{% if (rclone_vfs_cache_dir_lookup | length > 0) and item.vfs_cache.enabled %} +{% if (rclone_vfs_cache_dir_lookup | length > 0) and item.settings.vfs_cache.enabled %} --cache-dir={{ rclone_vfs_cache_dir_lookup }} \ {% endif %} --config={{ rclone_config_path }} \ - --buffer-size={{ '32M' if item.vfs_cache.enabled else '64M' }} \ + --buffer-size={{ '32M' if item.settings.vfs_cache.enabled else '64M' }} \ --dir-cache-time={{ rclone_cloud_dir_cache_time }} \ -{% if not item.vfs_cache.enabled %} +{% if not item.settings.vfs_cache.enabled %} --drive-chunk-size=64M \ {% endif %} --drive-pacer-burst=1000 \ @@ -43,17 +43,17 @@ ExecStart=/usr/bin/rclone mount \ --umask=002 \ --use-mmap \ --user-agent='{{ user_agent }}' \ -{% if item.vfs_cache.enabled %} +{% if item.settings.vfs_cache.enabled %} --vfs-cache-min-free-space={{ rclone_vfs_cache_min_free_space }} \ - --vfs-cache-max-age={{ item.vfs_cache.max_age | default('504h') }} \ - --vfs-cache-max-size={{ item.vfs_cache.size | default('50G') }} \ + --vfs-cache-max-age={{ item.settings.vfs_cache.max_age | default('504h') }} \ + --vfs-cache-max-size={{ item.settings.vfs_cache.size | default('50G') }} \ --vfs-cache-mode=full \ --vfs-cache-poll-interval={{ rclone_vfs_cache_poll_interval }} \ --vfs-fast-fingerprint \ --vfs-read-ahead=128M \ {% endif %} --vfs-read-chunk-size-limit=2G \ - --vfs-read-chunk-size={{ '32M' if item.vfs_cache.enabled else '64M' }} \ + --vfs-read-chunk-size={{ '32M' if item.settings.vfs_cache.enabled else '64M' }} \ -v \ "{{ rclone_remote_with_path }}" "/mnt/remote/{{ rclone_remote_name }}" ExecStop=/bin/fusermount3 -uz "/mnt/remote/{{ rclone_remote_name }}" @@ -61,7 +61,7 @@ Restart=on-abort RestartSec=5 StartLimitInterval=60s StartLimitBurst=3 -{% if item.vfs_cache.enabled %} +{% if item.settings.vfs_cache.enabled %} TimeoutSec=21600 LimitNOFILE=infinity LimitMEMLOCK=infinity diff --git a/roles/remote/templates/sftp.j2 b/roles/remote/templates/sftp.j2 index f089feff68..3c5d4ca44d 100644 --- a/roles/remote/templates/sftp.j2 +++ b/roles/remote/templates/sftp.j2 @@ -19,7 +19,7 @@ Type=notify ExecStartPre=/bin/sleep 10 ExecStart=/usr/bin/rclone mount \ --allow-other \ -{% if (rclone_vfs_cache_dir_lookup | length > 0) and item.vfs_cache.enabled %} +{% if (rclone_vfs_cache_dir_lookup | length > 0) and item.settings.vfs_cache.enabled %} --cache-dir={{ rclone_vfs_cache_dir_lookup }} \ {% endif %} --config={{ rclone_config_path }} \ @@ -35,10 +35,10 @@ ExecStart=/usr/bin/rclone mount \ --syslog \ --umask=002 \ --user-agent='{{ user_agent }}' \ -{% if item.vfs_cache.enabled %} +{% if item.settings.vfs_cache.enabled %} --vfs-cache-min-free-space={{ rclone_vfs_cache_min_free_space }} \ - --vfs-cache-max-age={{ item.vfs_cache.max_age | default('504h') }} \ - --vfs-cache-max-size={{ item.vfs_cache.size | default('50G') }} \ + --vfs-cache-max-age={{ item.settings.vfs_cache.max_age | default('504h') }} \ + --vfs-cache-max-size={{ item.settings.vfs_cache.size | default('50G') }} \ --vfs-cache-mode=full \ --vfs-cache-poll-interval={{ rclone_vfs_cache_poll_interval }} \ --vfs-fast-fingerprint \ diff --git a/roles/settings/tasks/subtasks/migrator/settings_yml/migration_01.yml b/roles/settings/tasks/subtasks/migrator/settings_yml/migration_01.yml index 39bb4eb461..c2b430299d 100644 --- a/roles/settings/tasks/subtasks/migrator/settings_yml/migration_01.yml +++ b/roles/settings/tasks/subtasks/migrator/settings_yml/migration_01.yml @@ -38,21 +38,21 @@ - name: Migrator | 'settings.yml' | Migration 01 | Create 'rclone.remotes' dict ansible.builtin.shell: | - yyq e -i '.rclone.remotes += [{"remote": "{{ rclone.remote }}", "template": "google", "upload": true, "upload_from": "/mnt/local/Media", "vfs_cache": {"enabled": false, "size": "50G", "max_age": "504h"}}]' {{ playbook_dir }}/{{ file }} + yyq e -i '.rclone.remotes += [{"remote": "{{ rclone.remote }}", "settings": {"mounts": true, "template": "google", "union": true, "upload": true, "upload_from": "/mnt/local/Media", "vfs_cache": {"enabled": false, "size": "50G", "max_age": "504h"}}}]' {{ playbook_dir }}/{{ file }} become: true become_user: "{{ saltbox_yml.stat.pw_name }}" when: (rclone.remotes is undefined) and rclone_old_remote_is_defined - name: Migrator | 'settings.yml' | Migration 01 | Create 'rclone.remotes' dict ansible.builtin.shell: | - yyq e -i '.rclone.remotes += [{"remote": "google", "template": "google", "upload": true, "upload_from": "/mnt/local/Media", "vfs_cache": {"enabled": false, "size": "50G", "max_age": "504h"}}]' {{ playbook_dir }}/{{ file }} + yyq e -i '.rclone.remotes += [{"remote": "google", "settings": {"mounts": true, "template": "google", "union": true, "upload": true, "upload_from": "/mnt/local/Media", "vfs_cache": {"enabled": false, "size": "50G", "max_age": "504h"}}}]' {{ playbook_dir }}/{{ file }} become: true become_user: "{{ saltbox_yml.stat.pw_name }}" when: (rclone.remotes is undefined) and not rclone_old_remote_is_defined - name: Migrator | 'settings.yml' | Migration 01 | Add feeder to remotes ansible.builtin.shell: | - yyq e -i '.rclone.remotes += [{"remote": "feeder", "template": "sftp", "upload": false, "upload_from": "/mnt/local/Media", "vfs_cache": {"enabled": false, "size": "50G", "max_age": "504h"}}]' {{ playbook_dir }}/{{ file }} + yyq e -i '.rclone.remotes += [{"remote": "feeder", "settings": {"mounts": true, "template": "sftp", "union": true, "upload": false, "upload_from": "/mnt/local/Media", "vfs_cache": {"enabled": false, "size": "50G", "max_age": "504h"}}}]' {{ playbook_dir }}/{{ file }} become: true become_user: "{{ saltbox_yml.stat.pw_name }}" when: old_feeder_mount @@ -64,6 +64,50 @@ become_user: "{{ saltbox_yml.stat.pw_name }}" when: (rclone.enabled is undefined) +- name: Migrator | 'settings.yml' | Migration 01 | Upgrade to new settings format + when: (rclone.remotes is defined) + block: + - name: Migrator | 'settings.yml' | Migration 01 | Read current config file + ansible.builtin.slurp: + src: /srv/git/saltbox/settings.yml + register: settings_config_content + + - name: Migrator | 'settings.yml' | Migration 01 | Parse the configuration + ansible.builtin.set_fact: + settings_config: "{{ settings_config_content['content'] | b64decode | from_yaml }}" + + - name: Migrator | 'settings.yml' | Migration 01 | Transform the data structure + ansible.builtin.set_fact: + new_remotes: "{{ new_remotes | default([]) + [{'remote': item.remote, 'settings': {'mount': true, 'template': item.template, 'union': true, 'upload': item.upload, 'upload_from': item.upload_from, 'vfs_cache': {'enabled': item.vfs_cache.enabled, 'max_age': item.vfs_cache.max_age, 'size': item.vfs_cache.size}}}] }}" + loop: "{{ settings_config.rclone.remotes }}" + when: item.template is defined + + - name: Migrator | 'settings.yml' | Migration 01 | Combine new structure with original config + ansible.builtin.set_fact: + new_config: "{{ (settings_config | combine({'rclone': {'remotes': new_remotes}}, recursive=True)) }}" + when: new_remotes is defined + + - name: Migrator | 'settings.yml' | Migration 01 | Write the new configuration file + ansible.builtin.copy: + dest: /srv/git/saltbox/settings.yml + content: "{{ new_config | to_nice_yaml }}" + owner: "{{ saltbox_yml.stat.uid }}" + group: "{{ saltbox_yml.stat.gid }}" + mode: "0664" + when: new_remotes is defined + +- name: Migrator | 'settings.yml' | Migration 01 | Convert 'true' to 'yes' + ansible.builtin.command: > + yyq eval ' + (.. | select(tag == "!!bool" and . == true)) |= "yes" + ' /srv/git/saltbox/settings.yml -i + +- name: Migrator | 'settings.yml' | Migration 01 | Convert 'false' to 'no' + ansible.builtin.command: > + yyq eval ' + (.. | select(tag == "!!bool" and . == false)) |= "no" + ' /srv/git/saltbox/settings.yml -i + - name: Migrator | 'settings.yml' | Migration 01 | Remove 'null' values ansible.builtin.replace: path: "{{ playbook_dir }}/{{ file }}" diff --git a/roles/unionfs/defaults/main.yml b/roles/unionfs/defaults/main.yml index a023ab1c17..0c0e301701 100644 --- a/roles/unionfs/defaults/main.yml +++ b/roles/unionfs/defaults/main.yml @@ -49,6 +49,9 @@ mergerfs_download_backup_url: " mergerfs_mount_branches: "{{ local_mount_branch }}{{ _remotes_list }}" +mergerfs_branch_mode: "NC" +mergerfs_remote_branch_mode_lookup: "{{ lookup('vars', 'mergerfs_remote_' + rclone_remote_name + '_branch_mode', default=mergerfs_branch_mode) }}" + mergerfs_mount_service_after: "network-online.target" mergerfs_mount_readdir_policy: "seq" diff --git a/roles/unionfs/tasks/subtasks/docker/daemon.yml b/roles/unionfs/tasks/subtasks/docker/daemon.yml index cf89b6525c..64b1a2c30b 100644 --- a/roles/unionfs/tasks/subtasks/docker/daemon.yml +++ b/roles/unionfs/tasks/subtasks/docker/daemon.yml @@ -11,7 +11,7 @@ ansible.builtin.set_fact: _remotes_services_list: "{{ _remotes_services_list | default('') + (rclone_service_template + rclone_remote_name + '.service ') }}" with_items: "{{ rclone.remotes }}" - when: rclone_remote_is_defined and use_remote and item.template != "nfs" + when: rclone_remote_is_defined and use_remote and item.settings.template != "nfs" and item.settings.mount - name: "Docker | Daemon | Create override directory" ansible.builtin.file: diff --git a/roles/unionfs/tasks/subtasks/variables.yml b/roles/unionfs/tasks/subtasks/variables.yml index c97b3d8afd..14dedd3cfa 100644 --- a/roles/unionfs/tasks/subtasks/variables.yml +++ b/roles/unionfs/tasks/subtasks/variables.yml @@ -12,9 +12,9 @@ ################################ - name: Variables | Set '_remotes_list' variable ansible.builtin.set_fact: - _remotes_list: "{{ _remotes_list | default('') + '/mnt/remote/' + (item.remote.split(':')[0] if ':' in item.remote else item.remote) + '=NC:' }}" + _remotes_list: "{{ _remotes_list | default('') + '/mnt/remote/' + (item.remote.split(':')[0] if ':' in item.remote else item.remote) + '=' + mergerfs_remote_branch_mode_lookup + ':' }}" with_items: "{{ rclone.remotes }}" - when: rclone_remote_is_defined and use_remote + when: rclone_remote_is_defined and use_remote and item.settings.mount and item.settings.union - name: Variables | Append 'custom_mount_branch' to '_remotes_list' variable ansible.builtin.set_fact: