ansible: move away from limited /tmp ramdisk, and clean up dir structure

.github/workflows/testbuild.yml

@@ -46,4 +46,4 @@ jobs:
         uses: actions/upload-artifact@v1
         with:
           name: ansibleimages
-          path: /tmp/ansible-openwrt/images
+          path: ./tmp/images

.gitignore

@@ -1,4 +1,5 @@
 */__pycache__/
 .vscode/
 venv/
-.env
+.env
+tmp/

FAQ.md

@@ -16,15 +16,15 @@ pip3 install -r requirements.txt
 ansible-playbook play.yml
 ```
 
-This will generate config files for all devices for inspection. The output path is `/tmp/ansible-openwrt/configs`.
+This will generate config files for all devices for inspection. The output path is `./tmp/configs`.
 
 ## How to spin up a config run and image generation?
 
 ```sh
 ansible-playbook play.yml --tags image
 ```
 
-The output path for the images is `/tmp/ansible-openwrt/images`.
+The output path for the images is `./tmp/images`.
 
 ## How to limit a config and image generation?
 
@@ -54,7 +54,7 @@ ansible-playbook play.yml --tags flash
 
 1. Create a location folder at `/group_vars/` and fill in at least `general.yml`, `networks.yml` and `owm.yml`.
 2. Create a folder for every OpenWrt device at the location under `/host_vars/`. Paste the `base.yml` in there.
-3. Run the image creation as shown in the commands above (image will be in `/tmp/ansible-openwrt/images/`).
+3. Run the image creation as shown in the commands above (image will be in `./tmp/images/`).
 4. Flash the image to your router.
 5. Secure the router by setting a root password using SSH or the web interface.
 6. Done!
@@ -80,11 +80,14 @@ ansible-inventory --graph
 ansible-playbook play.yml --tags image --limit example-core,example-ap1
 
 # Change into images directory
-cd /tmp/ansible-openwrt/images
+cd ./tmp/images
 
 # Optional: Keyscan for hostkeys
 for i in *.bin; do hostname="$(echo $i | awk -F '.' '{print $1}')"; ssh_target="$hostname.olsr"; ssh-keyscan "$ssh_target"; done
 
 # "oneliner" to mass-flash all devices where we have an image (use with caution)
-for i in *.bin; do hostname="$(echo $i | awk -F '.' '{print $1}')"; ssh_target="root@$hostname.olsr"; path="/tmp/$i"; echo -e "\e[92m$(date +%H:%M:%S) - $hostname: Disabling non-mesh wireless networks to free memory and sleep 13 seconds until change is applied (required for 32mb devices)\e[0m"; ssh "$ssh_target" "for i in \$(uci show wireless | grep mode=\'ap\' | awk -F '.' '{print \$2}'); do uci set wireless.\$i.disabled=1; done; uci commit wireless; ubus call uci reload_config;"; sleep 13; echo -e "\e[92m$(date +%H:%M:%S) - $hostname: Disabling unnecessary services to free even more memory\e[0m"; ssh "$ssh_target" "/etc/init.d/collectd stop; /etc/init.d/luci_statistics stop; /etc/init.d/sysntpd stop; /etc/init.d/urngd stop; /etc/init.d/rpcd stop; /etc/init.d/naywatch stop 2> /dev/null"; echo -e "\e[92m$(date +%H:%M:%S) - $hostname: Transfering image\e[0m"; scp -O "$i" "$ssh_target:$path"; echo -e "\e[92m$(date +%H:%M:%S) - $hostname: Start sysupgrade \e[0m"; ssh "$ssh_target" "sysupgrade $path" ; done
+for i in *.bin; do hostname="$(echo $i | awk -F '.' '{print $1}')"; ssh_target="root@$hostname.olsr"; path="/tmp/$i"; echo -e "\e[92m$(date +%H:%M:%S) - $hostname: Disabling non-mesh wireless networks to free memory and sleep 13 seconds until change is applied (required for 32mb devices)\e[0m";
+ssh "$ssh_target" "for i in \$(uci show wireless | grep mode=\'ap\' | awk -F '.' '{print \$2}'); do uci set wireless.\$i.disabled=1; done; uci commit wireless; ubus call uci reload_config;"; sleep 13; echo -e "\e[92m$(date +%H:%M:%S) - $hostname: Disabling unnecessary services to free even more memory\e[0m";
+ssh "$ssh_target" "/etc/init.d/collectd stop; /etc/init.d/luci_statistics stop; /etc/init.d/sysntpd stop; /etc/init.d/urngd stop; /etc/init.d/rpcd stop; /etc/init.d/naywatch stop 2> /dev/null"; echo -e "\e[92m$(date +%H:%M:%S) - $hostname: Transfering image\e[0m"; scp -O "$i" "$ssh_target:$path";
+echo -e "\e[92m$(date +%H:%M:%S) - $hostname: Start sysupgrade \e[0m"; ssh "$ssh_target" "sysupgrade $path" ; done
 ```

README.md

@@ -49,7 +49,7 @@ If you updated certain locations you can automaticly updating their wikiarticle
     ansible-playbook play.yml --tags wiki
 This works by automaticly replacing semantic values in the coresponding article resulting in the placeholders throughout the article being replaced with the new values. For an example you can have a look at the [wikiarticle of Fesev](https://wiki.freifunk.net/Berlin:Standorte:Fesev).
 To add this option to your wikiarticle add a section called "Konfiguration" and replace all values that you want to automaticly change as you can see in the example-article. If you want to add a new location you can start with [this template](https://wiki.freifunk.net/Berlin:Standorte:Template).
-Please note that for the first time you have to copy the wikiupdateroutput from /tmp/ansible-openwrt/wikiupdater/ into your acrticle or set the coordinates manually.
+Please note that for the first time you have to copy the wikiupdateroutput from ./tmp/wikiupdater/ into your acrticle or set the coordinates manually.
 
 ## Support Information
 

generate-images.sh

@@ -44,14 +44,14 @@ select location in "${locations[@]}"
     # generate all images if selected
     if [[ "$REPLY" == all ]] 
     then
-        ansible-playbook play.yml --tags image && echo "location of generated images: /tmp/ansible-openwrt/images"
+        ansible-playbook play.yml --tags image && echo "location of generated images: ./tmp/images"
         break
     fi
 
     # delete old directories and get rid of artifacts
     if [[ "$REPLY" == clean ]]
     then
-        [ -d "/tmp/ansible-openwrt/" ] && rm -r /tmp/ansible-openwrt/
+        [ -d "./tmp/" ] && rm -r ./tmp/
         echo "tmp directory was deleted..."
         continue
     fi
@@ -81,7 +81,7 @@ select location in "${locations[@]}"
     # generate images
     echo "firmwares for the following location will be generated:"
     echo "$location"
-    ansible-playbook play.yml --limit "location_$location" --tags image && echo "location of generated images: /tmp/ansible-openwrt/images"
+    ansible-playbook play.yml --limit "location_$location" --tags image && echo "location of generated images: ./tmp/images"
 
     # break the loop
     break

roles/cfg_openwrt/tasks/add_symlinks.yml

@@ -2,12 +2,12 @@
 - name: Create rc.d directory
   file:
     state: directory
-    path: "{{ config_path }}/etc/rc.d"
+    path: "{{ configs_dir }}/etc/rc.d"
     mode: "755"
 
 - name: Enable replace_secrets symlink
   file:
     src: "../init.d/replace_secrets"
-    dest: "{{ config_path }}/etc/rc.d/S15replace_secrets"
+    dest: "{{ configs_dir }}/etc/rc.d/S15replace_secrets"
     state: link
   when: role == "corerouter" or role == "ap"

roles/cfg_openwrt/tasks/fix_permissions.yml

@@ -1,22 +1,22 @@
 ---
 - name: Set permission for dropbear directory
   file:
-    path: "{{ config_path }}/etc/dropbear/"
+    path: "{{ configs_dir }}/etc/dropbear/"
     mode: "700"
 
 - name: Set permission for authorized_keys
   file:
-    path: "{{ config_path }}/etc/dropbear/authorized_keys"
+    path: "{{ configs_dir }}/etc/dropbear/authorized_keys"
     mode: "600"
 
 - name: Set permission for replace_secrets script
   file:
-    path: "{{ config_path }}/etc/init.d/replace_secrets"
+    path: "{{ configs_dir }}/etc/init.d/replace_secrets"
     mode: "755"
   when: role != "gateway"
 
 - name: Set permission for force_mcast_mld_reports.sh script
   file:
-    path: "{{ config_path }}/etc/force_mcast_mld_reports.sh"
+    path: "{{ configs_dir }}/etc/force_mcast_mld_reports.sh"
     mode: "755"
   when: role in ['corerouter', 'gateway']

roles/cfg_openwrt/tasks/imagebuilder.yml

@@ -2,33 +2,33 @@
 - name: Create images dir
   file:
     state: directory
-    path: "{{ image_dir }}"
+    path: "{{ images_dir }}"
     mode: "755"
   run_once: true
 - name: Clear old image and logfile
   file:
     state: absent
     path: "{{ item }}"
   loop:
-    - "{{ image_dir }}/{{ inventory_hostname }}.bin"
-    - "{{ image_dir }}/{{ inventory_hostname }}.img"
-    - "{{ image_dir }}/{{ inventory_hostname }}.log"
+    - "{{ images_dir }}/{{ inventory_hostname }}.bin"
+    - "{{ images_dir }}/{{ inventory_hostname }}.img"
+    - "{{ images_dir }}/{{ inventory_hostname }}.log"
 
 - name: Create Download directory
   file:
-    path: "{{ imagebuilder_dir }}"
+    path: "{{ dl_dir }}"
     state: directory
     mode: "755"
   run_once: true
 
 - name: Set destination path for imagebuilder
   set_fact:
-    imagebuilder_path: "{{ imagebuilder_dir }}/{{ imagebuilder | basename }}"
+    imagebuilder_path: "{{ dl_dir }}/{{ imagebuilder | basename }}"
 
 - name: Download Imagebuilder if upstream is newer
   get_url:
     url: "{{ imagebuilder }}"
-    dest: "{{ imagebuilder_dir }}/{{ imagebuilder_filename }}"
+    dest: "{{ dl_dir }}/{{ imagebuilder_filename }}"
     mode: "644"
   when: '"http" in imagebuilder'
 
@@ -37,30 +37,30 @@
     argv:
       - "cp"
       - "{{ imagebuilder }}"
-      - "{{ imagebuilder_dir }}"
+      - "{{ dl_dir }}"
   when: '"http" not in imagebuilder'
   changed_when: true
 
 - name: Clear workdir
   file:
     state: absent
-    path: "{{ imagebuild_dir }}"
+    path: "{{ build_dir }}"
 
 - name: (Re)-Create workdir
   file:
     state: directory
-    path: "{{ imagebuild_dir }}"
+    path: "{{ build_dir }}"
     mode: "755"
 
 - name: Extract imagebuilder
   unarchive:
     src: "{{ imagebuilder_path }}"
-    dest: "{{ imagebuild_dir }}"
+    dest: "{{ build_dir }}"
     extra_opts: ['--strip=1']
 
 - name: Read instruction set
   shell:
-    chdir: "{{ imagebuild_dir }}"
+    chdir: "{{ build_dir }}"
     cmd: grep -oP 'CONFIG_TARGET_ARCH_PACKAGES="\K[^"]+' .config
   register: instr_set_result
   when: 'instr_set is not defined and feed_version is defined'
@@ -73,70 +73,70 @@
 
 - name: Insert falter feed
   lineinfile:
-    path: "{{ imagebuild_dir }}/repositories.conf"
+    path: "{{ build_dir }}/repositories.conf"
     line: "{{ feed | replace('__INSTR_SET__', instr_set) | replace('__FEED_VERSION__', feed_version) }}"
   when: 'feed_version is defined'
 
 - name: Define Key-Dir
   stat:
-    path: "{{ imagebuild_dir }}/keys/"
+    path: "{{ build_dir }}/keys/"
   register: keydir
 
 - name: Add falter feed key
   copy:
     src: "files/packagefeed_master.pub"
-    dest: "{{ imagebuild_dir }}/keys/61a078a38408e710"  # matches fingerprint
+    dest: "{{ build_dir }}/keys/61a078a38408e710"  # matches fingerprint
     mode: "preserve"
   when: 'feed_version is defined and keydir.stat.exists'
 
 - name: Disable Signature verification if required
   lineinfile:
-    path: "{{ imagebuild_dir }}/repositories.conf"
+    path: "{{ build_dir }}/repositories.conf"
     line: "option check_signature"
     state: "absent"
   when: 'imagebuilder_disable_signature_check is defined and imagebuilder_disable_signature_check'
 
 - name: Override compat_version check to bbb-configs exclusive value 9.9
   lineinfile:
-    path: "{{ imagebuild_dir }}/include/image-commands.mk"
+    path: "{{ build_dir }}/include/image-commands.mk"
     search_string: "compat_version=$(if $(DEVICE_COMPAT_VERSION),$(DEVICE_COMPAT_VERSION),1.0)"
     line: "compat_version=9.9"
 
 - name: Run Imagebuilder
   changed_when: false
   command:
-    chdir: "{{ imagebuild_dir }}"
+    chdir: "{{ build_dir }}"
     argv:
       - "bash"
       - "-c"
-      - "make image PROFILE='{{ override_target | default(model) }}' PACKAGES='{{ packages | join(' ') }}' DISABLED_SERVICES='{{ disabled_services | join(' ') }}' FILES='{{ config_path }}' 1>&2"
+      - "make image PROFILE='{{ override_target | default(model) }}' PACKAGES='{{ packages | join(' ') }}' DISABLED_SERVICES='{{ disabled_services | join(' ') }}' FILES='{{ configs_dir }}' 1>&2"
   register: output
 
 - name: Write build log to output dir
   copy:
     content: "{{ output.stderr }}"
-    dest: "{{ image_dir }}/{{ inventory_hostname }}.log"
+    dest: "{{ images_dir }}/{{ inventory_hostname }}.log"
     mode: "preserve"
 
 - name: Find output image
   find:
-    paths: "{{ imagebuild_dir }}/bin/"
+    paths: "{{ build_dir }}/bin/"
     patterns: "{{ image_search_pattern }}"
     recurse: true
   register: image_list
 
 - name: Check Imagebuilder result
   assert:
     that: image_list.files | length > 0
-    fail_msg: 'No image was built, check log at {{ image_dir }}/{{ inventory_hostname }}.log'
+    fail_msg: 'No image was built, check log at {{ images_dir }}/{{ inventory_hostname }}.log'
 
 - name: Declare source
   set_fact:
     image_src: "{{ image_list.files | map(attribute='path') | list | first }}"
 
 - name: Declare target
   set_fact:
-    image: "{{ image_dir }}/{{ inventory_hostname }}{{ image_src | splitext | last }}"
+    image: "{{ images_dir }}/{{ inventory_hostname }}{{ image_src | splitext | last }}"
 
 - name: Copy Image to Image dir
   copy:

roles/cfg_openwrt/tasks/main.yml

@@ -15,21 +15,21 @@
 
 - name: Prepare some variables (1)
   set_fact:
-    basedir: /tmp/ansible-openwrt
+    basedir: "{{ playbook_dir }}/tmp"
   tags: always
 
 - name: Prepare some variables
   set_fact:
-    config_path: "{{ basedir }}/configs/{{ inventory_hostname }}"
-    imagebuilder_dir: "{{ basedir }}/imagebuilders/"
-    imagebuild_dir: "{{ basedir }}/tmp/imagebuild/{{ inventory_hostname }}"
-    image_dir: "{{ basedir }}/images/"
+    configs_dir: "{{ basedir }}/configs/{{ inventory_hostname }}"
+    dl_dir: "{{ basedir }}/dl/"
+    build_dir: "{{ basedir }}/build/{{ inventory_hostname }}"
+    images_dir: "{{ basedir }}/images/"
     wikiupdater_dir: "{{ basedir }}/wikiupdater/"
   tags: always
 
 - name: Create directory
   file:
-    path: "{{ config_path }}"
+    path: "{{ configs_dir }}"
     state: directory
     mode: "755"
   tags: always
@@ -51,23 +51,23 @@
 - name: Add poemgr config if poemgr target
   template:
     src: "{{ role_path }}/templates/common/config/poemgr.j2"
-    dest: "{{ config_path }}/etc/config/poemgr"
+    dest: "{{ configs_dir }}/etc/config/poemgr"
     mode: "644"
   when:
     - poemgr_ports is defined
   tags: always
 
 - name: "Make sure /lib/firmware is present"
   file:
-    dest: "{{ config_path }}/lib/firmware/ath10k"
+    dest: "{{ configs_dir }}/lib/firmware/ath10k"
     state: directory
     recurse: true
   when: ath10k_ct_fwcfg is defined
   tags: always
 
 - name: "Configure ath10k-ct driver"
   copy:
-    dest: "{{ config_path }}/lib/firmware/ath10k/fwcfg-{{ item.id }}.txt"
+    dest: "{{ configs_dir }}/lib/firmware/ath10k/fwcfg-{{ item.id }}.txt"
     content: "{{ item.config }}"
     mode: "644"
   loop: "{{ ath10k_ct_fwcfg | default([]) }}"

roles/cfg_openwrt/tasks/render_etc.yml

@@ -20,7 +20,7 @@
 - name: Create /etc directories for template output
   tags: always
   file:
-    path: "{{ config_path }}/etc/{{ item.path }}"
+    path: "{{ configs_dir }}/etc/{{ item.path }}"
     state: directory
     mode: "755"
   loop: "{{ lookup('community.general.filetree', role_path + '/templates/' + role) }}"
@@ -31,7 +31,7 @@
 - name: Create /etc directories for static files
   tags: always
   file:
-    path: "{{ config_path }}/etc/{{ item.path }}"
+    path: "{{ configs_dir }}/etc/{{ item.path }}"
     state: directory
     mode: "755"
   loop: "{{ lookup('community.general.filetree', role_path + '/files/' + role) }}"
@@ -43,7 +43,7 @@
   tags: always
   template:
     src: "{{ item.root }}/{{ item.path }}"
-    dest: "{{ config_path }}/etc/{{ item.path | regex_replace('.j2$', '') }}"
+    dest: "{{ configs_dir }}/etc/{{ item.path | regex_replace('.j2$', '') }}"
     mode: "644"
   loop: "{{ lookup('community.general.filetree', role_path + '/templates/' + role) }}"
   when: item.path is regex('\.j2$')
@@ -54,7 +54,7 @@
   tags: always
   copy:
     src: "{{ item.root }}/{{ item.path }}"
-    dest: "{{ config_path }}/etc/{{ item.path }}"
+    dest: "{{ configs_dir }}/etc/{{ item.path }}"
     mode: "preserve"
   loop: "{{ lookup('community.general.filetree', role_path + '/files/' + role) }}"
   when: (not item.path is regex('\.j2$')) and (item.state == 'file' or item.state == 'link')