From 51506f1c42dcd426cce5ab90afd73af74c75a9a1 Mon Sep 17 00:00:00 2001 From: Christopher Engelhard Date: Tue, 22 May 2018 12:04:16 +0200 Subject: [PATCH 01/12] Install/remove dependencies explicitly Furure-prrofing: Don't rely on groups/build-deps when source is not from the repos. Source dependencies might differ from repo dependencies/groups. This is the case when e.g. MariaDB in repo is a different branch than the source. --- ansible/roles/compiler/tasks/main.yml | 104 ++++++------------------- ansible/roles/unixbench/tasks/main.yml | 12 +-- 2 files changed, 30 insertions(+), 86 deletions(-) diff --git a/ansible/roles/compiler/tasks/main.yml b/ansible/roles/compiler/tasks/main.yml index af2d864..3840d5d 100644 --- a/ansible/roles/compiler/tasks/main.yml +++ b/ansible/roles/compiler/tasks/main.yml @@ -11,19 +11,27 @@ dest: /tmp copy: no -- name: set source repo - apt_repository: - repo: "deb-src http://archive.ubuntu.com/ubuntu/ xenial-updates universe" - state: present - -- name: install packages need to build mariadb +- name: install packages needed to build mariadb apt: name: "{{item}}" - state: build-dep + state: present update_cache: yes install_recommends: no with_items: - - mariadb-server + - bison + - make + - cmake + - g++ + - gcc + - git + - libaio-dev + - libcurl4-gnutls-dev + - libevent-dev + - libgnutls28-dev + - libjemalloc-dev + - libncurses-dev + - libxml2-dev + - zlib1g-dev - name: configure the mariadb build shell: cmake . -DBUILD_CONFIG=mysql_release @@ -57,83 +65,17 @@ purge: yes state: absent with_items: - - autotools-dev - - binutils - bison - - build-essential - - chrpath + - make - cmake - - cmake-data - - cpp - - cpp-5 - - debhelper - - dh-apparmor - - dh-strip-nondeterminism - - dpkg-dev - g++ - - g++-5 - gcc - - gcc-5 - - gdb - - gettext - - icu-devtools - - intltool-debian + - git - libaio-dev - - libaio1 - - libarchive-zip-perl - - libarchive13 - - libasan2 - - libatomic1 - - libbabeltrace-ctf1 - - libbabeltrace1 - - libbison-dev - - libboost-dev - - libboost1.58-dev - - libc-dev-bin - - libc6-dev - - libcc1-0 - - libcilkrts5 - - libcroco3 - - libcurl3 - - libdpkg-perl - - libfile-stripnondeterminism-perl - - libgcc-5-dev - - libgomp1 - - libicu-dev - - libisl15 - - libitm1 + - libcurl4-gnutls-dev + - libevent-dev + - libgnutls28-dev - libjemalloc-dev - - libjemalloc1 - - libjsoncpp1 - - libjudy-dev - - libjudydebian1 - - liblsan0 - - libltdl-dev - - libltdl7 - - libmpc3 - - libmpx0 - - libncurses5-dev - - libodbc1 - - libpam0g-dev - - libpcre16-3 - - libpcre3-dev - - libpcre32-3 - - libpcrecpp0v5 - - libquadmath0 - - libreadline-gplv2-dev - - libstdc++-5-dev - - libtimedate-perl - - libtinfo-dev - - libtsan0 - - libubsan0 - - libunistring0 + - libncurses-dev - libxml2-dev - - linux-libc-dev - - m4 - - make - - odbcinst - - odbcinst1debian2 - - po-debconf - - unixodbc - - unixodbc-dev - - zlib1g-dev + - zlib1g-dev diff --git a/ansible/roles/unixbench/tasks/main.yml b/ansible/roles/unixbench/tasks/main.yml index abe43d6..0f06b98 100644 --- a/ansible/roles/unixbench/tasks/main.yml +++ b/ansible/roles/unixbench/tasks/main.yml @@ -7,8 +7,9 @@ update_cache: yes install_recommends: no with_items: - - gcc-5 - - build-essential + - gcc + - libc-dev + - make - name: download UnixBench get_url: @@ -40,13 +41,14 @@ dest: ../logs/{{inventory_hostname}}/unixbench.log flat: yes -- name: remove installed packages +- name: remove installed packages apt: name: "{{item}}" autoremove: yes purge: yes state: absent with_items: - - gcc-5 - - build-essential + - gcc + - libc-dev + - make From e5355f11a183516ccd2eb4d86e0b7383dec6df27 Mon Sep 17 00:00:00 2001 From: Christopher Engelhard Date: Tue, 22 May 2018 12:10:00 +0200 Subject: [PATCH 02/12] Set ansible_python_interpreter automatically On systems that don't have /usr/bin/python, use /usr/bin/python3. Call /usr/bin/python(3) instead of /usr/bin/env python in speedtest-cli because 'python' might not be present. --- ansible/roles/speedtest/tasks/main.yml | 6 ++++++ ansible/site.yml | 20 +++++++++++++++----- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/ansible/roles/speedtest/tasks/main.yml b/ansible/roles/speedtest/tasks/main.yml index 5943545..bf88e94 100644 --- a/ansible/roles/speedtest/tasks/main.yml +++ b/ansible/roles/speedtest/tasks/main.yml @@ -5,6 +5,12 @@ url: https://raw.githubusercontent.com/sivel/speedtest-cli/master/speedtest.py dest: /tmp/speedtest-cli +- name: set python executable + replace: + path: /tmp/speedtest-cli + regexp: '#!\/usr\/bin\/env python' + replace: "#!{{ ansible_python_interpreter }}" + - name: set execution permission file: path: /tmp/speedtest-cli diff --git a/ansible/site.yml b/ansible/site.yml index 76d116a..a9cc4f2 100644 --- a/ansible/site.yml +++ b/ansible/site.yml @@ -1,12 +1,22 @@ - hosts: all become: yes become_method: sudo + strategy: linear # this is the default. Set to 'free' to let hosts to run independently + gather_facts: false # needs to be run after ansible_python_interpreter is set up correctly - # pre_tasks: - # - name: Install python for Ansible on Ubuntu - # raw: test -e /usr/bin/python || (apt -y update && apt install -y python-minimal) - # when: ansible_distribution == "Ubuntu" - # changed_when: False + # set /usr/bin/python3 as interpreter on systems that don't ship /usr/bin/python + pre_tasks: + - name: detect available python interpreter + raw: which /usr/bin/python || which /usr/bin/python3 + register: py_interpreter + changed_when: py_interpreter.stdout != "" + - name: set python interpreter + set_fact: + ansible_python_interpreter: "{{ py_interpreter.stdout_lines[0] }}" + - debug: + msg: "{{ inventory_hostname }} python interpreter: {{ ansible_python_interpreter }}" + - name: Gathering Facts + setup: # AKA gather_facts roles: - { role: lynis, tags: ['security', 'lynis', 'info']} From 8eb34607033d8398d9f6102efd27e7a743cf1fbc Mon Sep 17 00:00:00 2001 From: Christopher Engelhard Date: Tue, 22 May 2018 12:15:52 +0200 Subject: [PATCH 03/12] Remove ansible_python_interpreter=.. --- ansible/hosts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ansible/hosts b/ansible/hosts index 22992c3..6e53375 100644 --- a/ansible/hosts +++ b/ansible/hosts @@ -1,18 +1,18 @@ [ovh] -ovh-vpsssd1-ubuntu ansible_host=X.X.X.X ansible_python_interpreter=/usr/bin/python3 +ovh-vpsssd1-ubuntu ansible_host=X.X.X.X [linode] -linode-linode1024-ubuntu ansible_host=X.X.X.X ansible_python_interpreter=/usr/bin/python3 +linode-linode1024-ubuntu ansible_host=X.X.X.X [digitalocean] -do-5bucks-ubuntu ansible_host=X.X.X.X ansible_python_interpreter=/usr/bin/python3 +do-5bucks-ubuntu ansible_host=X.X.X.X [scaleway] -scaleway-vc1s-ubuntu ansible_host=X.X.X.X ansible_python_interpreter=/usr/bin/python3 +scaleway-vc1s-ubuntu ansible_host=X.X.X.X [vultr] -vultr-20gbssd-ubuntu ansible_host=X.X.X.X ansible_python_interpreter=/usr/bin/python3 -vultr-25gbssd-ubuntu ansible_host=X.X.X.X ansible_python_interpreter=/usr/bin/python3 +vultr-20gbssd-ubuntu ansible_host=X.X.X.X +vultr-25gbssd-ubuntu ansible_host=X.X.X.X # vars by provider From 55edea048fb5f45b5a300bcc4daea74a2c50ce3b Mon Sep 17 00:00:00 2001 From: Christopher Engelhard Date: Tue, 22 May 2018 12:19:26 +0200 Subject: [PATCH 04/12] Add SSH pipelining --- ansible/ansible.cfg | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ansible/ansible.cfg b/ansible/ansible.cfg index 009dbb4..33e048b 100644 --- a/ansible/ansible.cfg +++ b/ansible/ansible.cfg @@ -1,2 +1,8 @@ [defaults] -inventory = hosts \ No newline at end of file +inventory = hosts +stdout_callback = yaml # makes error output more readable + +[ssh_connection] +pipelining = True +# enable this if ansible hangs on long-running tasks +#ssh_args = -o ServerAliveInterval=100 -o ControlMaster=auto -o ControlPersist=60m From d1ca495cecfd952c3ded12d7f3c70343e8a81e71 Mon Sep 17 00:00:00 2001 From: Christopher Engelhard Date: Tue, 22 May 2018 12:20:04 +0200 Subject: [PATCH 05/12] Use downloads.mariadb.com instead of mirror Many mirrors don't keep older source version around. downloads.mariadb.com does. --- ansible/roles/compiler/tasks/main.yml | 16 ++++++++-------- ansible/site.yml | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ansible/roles/compiler/tasks/main.yml b/ansible/roles/compiler/tasks/main.yml index 3840d5d..b2ce8c0 100644 --- a/ansible/roles/compiler/tasks/main.yml +++ b/ansible/roles/compiler/tasks/main.yml @@ -2,12 +2,12 @@ - name: download mariadb get_url: - url: https://downloads.mariadb.org/f/mariadb-10.1.22/source/mariadb-10.1.22.tar.gz?serve - dest: /tmp/mariadb.tar.gz + url: https://downloads.mariadb.com/MariaDB/mariadb-{{mariadb_version}}/source/mariadb-{{mariadb_version}}.tar.gz + dest: /tmp/mariadb-{{mariadb_version}}.tar.gz - name: uncompress mariadb unarchive: - src: /tmp/mariadb.tar.gz + src: /tmp/mariadb-{{mariadb_version}}.tar.gz dest: /tmp copy: no @@ -36,13 +36,13 @@ - name: configure the mariadb build shell: cmake . -DBUILD_CONFIG=mysql_release args: - chdir: /tmp/mariadb-10.1.22 + chdir: /tmp/mariadb-{{mariadb_version}} - name: compile mariadb shell: (time -p make) > /tmp/compiler.log 2>&1 args: executable: /bin/bash - chdir: /tmp/mariadb-10.1.22 + chdir: /tmp/mariadb-{{mariadb_version}} - name: retrieve log fetch: @@ -52,11 +52,11 @@ - name: remove mariadb files file: - path: /tmp/test + path: "{{item}}" state: absent with_items: - - /tmp/mariadb-10.1.22 - - /tmp/mariadb.tar.gz + - /tmp/mariadb-{{mariadb_version}} + - /tmp/mariadb-{{mariadb_version}}.tar.gz - name: remove installed packages apt: diff --git a/ansible/site.yml b/ansible/site.yml index a9cc4f2..93bfc3b 100644 --- a/ansible/site.yml +++ b/ansible/site.yml @@ -28,7 +28,7 @@ - { role: fio, tags: ['fio', 'disk', 'benchmark']} - { role: speedtest, tags: ['speedtest', 'network', 'benchmark']} - { role: downloads, tags: ['downloads', 'network', 'benchmark']} #caution! ~93Gbits - - { role: compiler, tags: ['compiler', 'benchmark']} + - { role: compiler, tags: ['compiler', 'benchmark'], vars: { mariadb_version: 10.2.14 }} - { role: transcode, tags: ['transcode', 'multimedia', 'benchmark']} - { role: web, tags: ['web', 'benchmark']} - { role: common, tags: ['common']} From 5b7bf5ae69d6a3d24a57dc3ae2a269572680a833 Mon Sep 17 00:00:00 2001 From: Christopher Engelhard Date: Tue, 22 May 2018 12:22:18 +0200 Subject: [PATCH 06/12] Fix gcc segfault with mroonga db Some versions of gcc (currently those distributed with Ubuntu Bionic and Debian Stretch) segfault when compiling mroonga db, so omit it. Compiling still is sufficiently complex without it. --- ansible/roles/compiler/tasks/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ansible/roles/compiler/tasks/main.yml b/ansible/roles/compiler/tasks/main.yml index b2ce8c0..9146dd7 100644 --- a/ansible/roles/compiler/tasks/main.yml +++ b/ansible/roles/compiler/tasks/main.yml @@ -34,9 +34,10 @@ - zlib1g-dev - name: configure the mariadb build - shell: cmake . -DBUILD_CONFIG=mysql_release + shell: cmake . -DBUILD_CONFIG=mysql_release -DWITHOUT_MROONGA_STORAGE_ENGINE=YES args: chdir: /tmp/mariadb-{{mariadb_version}} + # some versions of gcc segfault when building mroonga, so omit it - name: compile mariadb shell: (time -p make) > /tmp/compiler.log 2>&1 From 1a0bd4ec9535bdacf0f788278b36aa83ff57250d Mon Sep 17 00:00:00 2001 From: Christopher Engelhard Date: Tue, 22 May 2018 12:26:27 +0200 Subject: [PATCH 07/12] Fix incomplete removal of mysql-* packages/files --- ansible/roles/sysbench/tasks/main.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/ansible/roles/sysbench/tasks/main.yml b/ansible/roles/sysbench/tasks/main.yml index 4e5de9e..b2d764e 100644 --- a/ansible/roles/sysbench/tasks/main.yml +++ b/ansible/roles/sysbench/tasks/main.yml @@ -133,7 +133,17 @@ - name: remove mysql apt: - name: mysql-server + name: 'mysql*' autoremove: yes purge: yes state: absent + +- name: remove remaining mysql files + file: + path: "{{ item }}" + state: absent + with_items: + - /var/lib/mysql + - /var/lib/mysql-files + - /var/lib/mysql-keyring + - /etc/mysql From bc51566af535c57f05af909cfa057c12b15d884d Mon Sep 17 00:00:00 2001 From: Christopher Engelhard Date: Tue, 22 May 2018 12:27:45 +0200 Subject: [PATCH 08/12] Use sysbench 1.x from upstream repo Syntax between sysbench 1.x and 0.4.x is very different. Using sysbench 1.x from upstream directly ensures consistency regardless of underlying distribution. --- ansible/roles/sysbench/tasks/main.yml | 46 ++++++++++++++++++--------- 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/ansible/roles/sysbench/tasks/main.yml b/ansible/roles/sysbench/tasks/main.yml index b2d764e..449350e 100644 --- a/ansible/roles/sysbench/tasks/main.yml +++ b/ansible/roles/sysbench/tasks/main.yml @@ -1,5 +1,27 @@ --- +- name: get signing key for sysbench repository + get_url: + url: https://packagecloud.io/akopytov/sysbench/gpgkey + dest: /tmp/sysbench-gpgkey + +- name: install signing key + shell: apt-key add /tmp/sysbench-gpgkey &>/dev/null + +- name: add apt https support + apt: + name: apt-transport-https + state: latest + update_cache: yes + install_recommends: no + +- name: add sysbench repository + apt_repository: + repo: "deb https://packagecloud.io/akopytov/sysbench/{{ ansible_distribution | lower }}/ {{ ansible_distribution_release }} main" + state: present + filename: sysbench + when: ansible_distribution_release != "bionic" # bionic already has speedtest 1.x and pc currently does not have a bionic repo + - name: install sysbench apt: name: sysbench @@ -8,7 +30,7 @@ install_recommends: no - name: run sysbench cpu - shell: sysbench --test=cpu --cpu-max-prime=20000 --num-threads=1 run >> /tmp/sysbench_cpu.log + shell: sysbench cpu --cpu-max-prime=20000 --num-threads=1 run >> /tmp/sysbench_cpu.log args: chdir: /tmp executable: /bin/bash @@ -21,14 +43,14 @@ flat: yes - name: run sysbench memory random read - shell: sysbench --test=memory --memory-access-mode=rnd --memory-oper=read --num-threads=1 run >> /tmp/sysbench_ram_randrd.log + shell: sysbench memory --memory-access-mode=rnd --memory-oper=read --num-threads=1 run >> /tmp/sysbench_ram_randrd.log args: chdir: /tmp executable: /bin/bash with_sequence: count=5 - name: run sysbench memory random write - shell: sysbench --test=memory --memory-access-mode=rnd --memory-oper=write --num-threads=1 run >> /tmp/sysbench_ram_randrw.log + shell: sysbench memory --memory-access-mode=rnd --memory-oper=write --num-threads=1 run >> /tmp/sysbench_ram_randrw.log args: chdir: /tmp executable: /bin/bash @@ -44,26 +66,26 @@ - sysbench_ram_randrw - name: prepare sysbench fileio - shell: sysbench --test=fileio --file-total-size=8G prepare + shell: sysbench fileio --file-total-size=8G prepare args: chdir: /tmp - name: run sysbench fileio random write - shell: sysbench --test=fileio --file-total-size=8G --file-test-mode=rndrw --init-rng=on --max-time=300 --max-requests=0 --file-block-size=4K run >> /tmp/sysbench_randrw.log + shell: sysbench fileio --file-total-size=8G --file-test-mode=rndrw --rand-seed=0 --max-time=300 --max-requests=0 --file-block-size=4K run >> /tmp/sysbench_randrw.log args: chdir: /tmp executable: /bin/bash with_sequence: count=5 - name: run sysbench fileio random read - shell: sysbench --test=fileio --file-total-size=8G --file-test-mode=rndrd --init-rng=on --max-time=300 --max-requests=0 --file-block-size=4K run >> /tmp/sysbench_randrd.log + shell: sysbench fileio --file-total-size=8G --file-test-mode=rndrd --rand-seed=0 --max-time=300 --max-requests=0 --file-block-size=4K run >> /tmp/sysbench_randrd.log args: chdir: /tmp executable: /bin/bash with_sequence: count=5 - name: cleanup sysbench fileio - shell: sysbench --test=fileio --file-total-size=8G cleanup + shell: sysbench fileio --file-total-size=8G cleanup args: chdir: /tmp @@ -102,17 +124,11 @@ - name: create sysbench database shell: mysql -u root -ppassword -e "CREATE DATABASE sysbench;" -- name: create database user - shell: mysql -u root -ppassword -e "CREATE USER 'sysbench'@'localhost' IDENTIFIED BY 'password';" - -- name: set database user privileges - shell: mysql -u root -ppassword -e "GRANT ALL PRIVILEGES ON *.* TO 'sysbench'@'localhost' IDENTIFIED BY 'password';" - - name: prepare sysbench database - shell: sysbench --test=oltp --db-driver=mysql --oltp-table-size=10000000 --mysql-db=sysbench --mysql-user=sysbench --mysql-password=password prepare + shell: sysbench oltp_read_only --db-driver=mysql --table_size=10000000 --mysql-db=sysbench --mysql-user=root --mysql-password=password prepare - name: run sysbench oltp - shell: sysbench --test=oltp --oltp-table-size=10000000 --mysql-db=sysbench --mysql-user=root --mysql-password=password --max-time=60 --max-requests=0 --num-threads=1 --oltp-reconnect-mode=random run >> /tmp/sysbench_mysql.log + shell: sysbench oltp_read_only --db-driver=mysql --table_size=10000000 --mysql-db=sysbench --mysql-user=root --mysql-password=password --time=60 --max-requests=0 --num-threads=1 run >> /tmp/sysbench_mysql.log args: chdir: /tmp executable: /bin/bash From 7ce080a051c0337fd21a1e420a39fd70169c3134 Mon Sep 17 00:00:00 2001 From: Christopher Engelhard Date: Tue, 22 May 2018 12:30:10 +0200 Subject: [PATCH 09/12] Add backports repo for ffmpeg support in Debian/Jessie --- ansible/roles/transcode/tasks/main.yml | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/ansible/roles/transcode/tasks/main.yml b/ansible/roles/transcode/tasks/main.yml index 28382b1..cfc3719 100644 --- a/ansible/roles/transcode/tasks/main.yml +++ b/ansible/roles/transcode/tasks/main.yml @@ -8,7 +8,14 @@ get_url: url: http://distribution.bbb3d.renderfarming.net/video/mp4/bbb_sunflower_2160p_30fps_normal.mp4 dest: /tmp/video4k.mp4 - when: video_down.stat.exists == False + when: video_down.stat.exists == False + +- name: add backports repository for ffmpeg on debian/jessie + apt_repository: + repo: "deb http://http.debian.net/debian/ jessie-backports main" + state: present + filename: jessie-backports + when: ansible_distribution_release == "jessie" - name: install ffmpeg apt: @@ -37,11 +44,15 @@ state: absent - name: remove videos - file: + file: path: "{{item}}" state: absent with_items: - /tmp/video4k.mp4 - /tmp/video_1080p.mp4 - +- name: remove backports repository on debian/jessie + apt_repository: + repo: "deb http://http.debian.net/debian/ jessie-backports main" + state: absent + when: ansible_distribution_release == "jessie" From a997aa5b98da7a8647f46218e0df8e0ef30265b1 Mon Sep 17 00:00:00 2001 From: Christopher Engelhard Date: Tue, 22 May 2018 12:31:11 +0200 Subject: [PATCH 10/12] Fix jordi/ab syntax --- ansible/roles/web/tasks/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/roles/web/tasks/main.yml b/ansible/roles/web/tasks/main.yml index 7582d9e..c86b497 100644 --- a/ansible/roles/web/tasks/main.yml +++ b/ansible/roles/web/tasks/main.yml @@ -98,7 +98,7 @@ when: wordpress_installed.rc == 0 - name: run ApacheBench app in local Docker against WordPress with 10 connections - shell: docker run --rm jordi/ab ab -t180 -c10 \ + shell: docker run --rm jordi/ab -t180 -c10 \ http://{{ansible_host}}/wordpress/{{ansible_date_time.year}}/{{ansible_date_time.month}}/{{ansible_date_time.day}}/hello-world/ > web_wp_ab_10.log delegate_to: localhost become: no @@ -114,7 +114,7 @@ when: wordpress_installed.rc == 0 - name: run ApacheBench app in local Docker against WordPress with 25 connections - shell: docker run --rm jordi/ab ab -t180 -c25 \ + shell: docker run --rm jordi/ab -t180 -c25 \ http://{{ansible_host}}/wordpress/{{ansible_date_time.year}}/{{ansible_date_time.month}}/{{ansible_date_time.day}}/hello-world/ > web_wp_ab_25.log delegate_to: localhost become: no @@ -130,7 +130,7 @@ when: wordpress_installed.rc == 0 - name: run ApacheBench app in local Docker against WordPress with 50 connections - shell: docker run --rm jordi/ab ab -t180 -c50 \ + shell: docker run --rm jordi/ab -t180 -c50 \ http://{{ansible_host}}/wordpress/{{ansible_date_time.year}}/{{ansible_date_time.month}}/{{ansible_date_time.day}}/hello-world/ > web_wp_ab_50.log delegate_to: localhost become: no From 3905ece28fbaef831160cbd51506c5b921b22848 Mon Sep 17 00:00:00 2001 From: Christopher Engelhard Date: Tue, 22 May 2018 12:38:10 +0200 Subject: [PATCH 11/12] Add logfile checksumming Calculate sha512-checksums for all files saved to ../log. Checksum of are saved as .sha512 . Unless the target file is IP-filtered or generated on localhost, checksumming is performed on the remote host. --- ansible/roles/common/tasks/main.yml | 17 ++++++ ansible/roles/compiler/tasks/main.yml | 15 +++++ ansible/roles/dd/tasks/main.yml | 37 ++++++++---- ansible/roles/downloads/tasks/main.yml | 17 +++++- ansible/roles/fio/tasks/main.yml | 39 ++++++++---- ansible/roles/hardware/tasks/main.yml | 83 +++++++++++--------------- ansible/roles/lynis/tasks/main.yml | 27 +++++++-- ansible/roles/ports/tasks/main.yml | 25 ++++++++ ansible/roles/speedtest/tasks/main.yml | 38 ++++++++++-- ansible/roles/sysbench/tasks/main.yml | 69 +++++++++++---------- ansible/roles/transcode/tasks/main.yml | 15 +++++ ansible/roles/unixbench/tasks/main.yml | 17 +++++- ansible/roles/web/tasks/main.yml | 29 +++++++++ 13 files changed, 317 insertions(+), 111 deletions(-) diff --git a/ansible/roles/common/tasks/main.yml b/ansible/roles/common/tasks/main.yml index c5db29c..7b02740 100644 --- a/ansible/roles/common/tasks/main.yml +++ b/ansible/roles/common/tasks/main.yml @@ -6,3 +6,20 @@ become: no tags: - gather + +- name: calculate sha512 checksum of results table + stat: + checksum_algorithm: sha512 + get_checksum: true + get_attributes: false + get_md5: false + get_mime: false + path: ../logs/tables.org + register: logfile + delegate_to: localhost + become: no + +- name: save checksum to file + local_action: + copy content="{{logfile.stat.checksum}}" dest="../logs/tables.org.sha512" + become: no diff --git a/ansible/roles/compiler/tasks/main.yml b/ansible/roles/compiler/tasks/main.yml index 9146dd7..f02e764 100644 --- a/ansible/roles/compiler/tasks/main.yml +++ b/ansible/roles/compiler/tasks/main.yml @@ -45,6 +45,21 @@ executable: /bin/bash chdir: /tmp/mariadb-{{mariadb_version}} +- name: calculate sha512 checksum of logfile + stat: + checksum_algorithm: sha512 + get_checksum: true + get_attributes: false + get_md5: false + get_mime: false + path: /tmp/compiler.log + register: logfile + +- name: save checksum to file + local_action: + copy content="{{ logfile.stat.checksum }}" dest="../logs/{{inventory_hostname}}/compiler.log.sha512" + become: no + - name: retrieve log fetch: src: /tmp/compiler.log diff --git a/ansible/roles/dd/tasks/main.yml b/ansible/roles/dd/tasks/main.yml index 339bb1b..799585e 100644 --- a/ansible/roles/dd/tasks/main.yml +++ b/ansible/roles/dd/tasks/main.yml @@ -6,26 +6,43 @@ chdir: /tmp with_sequence: count=5 -- name: retrieve log - fetch: - src: /tmp/dd_cpu.log - dest: ../logs/{{inventory_hostname}}/dd_cpu.log - flat: yes - - name: run dd IO test shell: dd if=/dev/zero of=test bs=64k count=16k conv=fdatasync 2>> dd_io.log args: chdir: /tmp with_sequence: count=5 -- name: retrieve log +- name: calculate sha512 checksums of logfiles + stat: + checksum_algorithm: sha512 + get_checksum: true + get_attributes: false + get_md5: false + get_mime: false + path: /tmp/{{item}}.log + register: logfile + with_items: + - dd_io + - dd_cpu + +- name: save checksums to files + local_action: + copy content="{{item.stat.checksum}}" dest="../logs/{{inventory_hostname}}/{{item.item}}.log.sha512" + become: no + with_items: + - "{{ logfile.results }}" + +- name: retrieve logs fetch: - src: /tmp/dd_io.log - dest: ../logs/{{inventory_hostname}}/dd_io.log + src: /tmp/{{item}}.log + dest: ../logs/{{inventory_hostname}}/{{item}}.log flat: yes + with_items: + - dd_io + - dd_cpu - name: remove auxiliary file - file: + file: path: /tmp/test state: absent diff --git a/ansible/roles/downloads/tasks/main.yml b/ansible/roles/downloads/tasks/main.yml index 080f5c3..9a5da58 100644 --- a/ansible/roles/downloads/tasks/main.yml +++ b/ansible/roles/downloads/tasks/main.yml @@ -7,7 +7,22 @@ shell: cat downloads_1.log downloads_2.log downloads_3.log > downloads.log args: chdir: /tmp - + +- name: calculate sha512 checksum of logfile + stat: + checksum_algorithm: sha512 + get_checksum: true + get_attributes: false + get_md5: false + get_mime: false + path: /tmp/downloads.log + register: logfile + +- name: save checksum to file + local_action: + copy content="{{ logfile.stat.checksum }}" dest="../logs/{{inventory_hostname}}/downloads.log.sha512" + become: no + - name: retrieve log fetch: src: /tmp/downloads.log diff --git a/ansible/roles/fio/tasks/main.yml b/ansible/roles/fio/tasks/main.yml index e5e39b5..605b910 100644 --- a/ansible/roles/fio/tasks/main.yml +++ b/ansible/roles/fio/tasks/main.yml @@ -13,14 +13,8 @@ chdir: /tmp with_sequence: count=5 -- name: retrieve log - fetch: - src: /tmp/fio_randread.log - dest: ../logs/{{inventory_hostname}}/fio_randread.log - flat: yes - - name: remove auxiliary files - file: + file: path: "{{item}}" state: absent with_items: @@ -39,14 +33,37 @@ chdir: /tmp with_sequence: count=5 -- name: retrieve log +- name: calculate sha512 checksums of logfiles + stat: + checksum_algorithm: sha512 + get_checksum: true + get_attributes: false + get_md5: false + get_mime: false + path: /tmp/{{item}}.log + register: logfile + with_items: + - fio_randwrite + - fio_randread + +- name: save checksums to files + local_action: + copy content="{{item.stat.checksum}}" dest="../logs/{{inventory_hostname}}/{{item.item}}.log.sha512" + become: no + with_items: + - "{{ logfile.results }}" + +- name: retrieve logs fetch: - src: /tmp/fio_randwrite.log - dest: ../logs/{{inventory_hostname}}/fio_randwrite.log + src: /tmp/{{item}}.log + dest: ../logs/{{inventory_hostname}}/{{item}}.log flat: yes + with_items: + - fio_randwrite + - fio_randread - name: remove auxiliary files - file: + file: path: "{{item}}" state: absent with_items: diff --git a/ansible/roles/hardware/tasks/main.yml b/ansible/roles/hardware/tasks/main.yml index 77c9684..f774b73 100644 --- a/ansible/roles/hardware/tasks/main.yml +++ b/ansible/roles/hardware/tasks/main.yml @@ -17,77 +17,74 @@ args: chdir: /tmp -- name: retrieve dmidecode log - fetch: - src: /tmp/hard_dmidecode.log - dest: ../logs/{{inventory_hostname}}/hard_dmidecode.log - flat: yes - - name: detect virtualization technology shell: systemd-detect-virt > hard_virt_tech.log args: chdir: /tmp -- name: retrieve vir-tech log - fetch: - src: /tmp/hard_virt_tech.log - dest: ../logs/{{inventory_hostname}}/hard_virt_tech.log - flat: yes - - name: run lscpu shell: lscpu > hard_lscpu.log args: chdir: /tmp -- name: retrieve lscpu log - fetch: - src: /tmp/hard_lscpu.log - dest: ../logs/{{inventory_hostname}}/hard_lscpu.log - flat: yes - - name: run lsblk shell: lsblk -p > hard_lsblk.log args: chdir: /tmp -- name: retrieve lsblk log - fetch: - src: /tmp/hard_lsblk.log - dest: ../logs/{{inventory_hostname}}/hard_lsblk.log - flat: yes - - name: run lshw shell: lshw > hard_lshw.log args: chdir: /tmp -- name: retrieve lshw log - fetch: - src: /tmp/hard_lshw.log - dest: ../logs/{{inventory_hostname}}/hard_lshw.log - flat: yes - - name: run inxi shell: inxi -Fxxx -c0 > hard_inxi.log args: chdir: /tmp -- name: retrieve inxi log - fetch: - src: /tmp/hard_inxi.log - dest: ../logs/{{inventory_hostname}}/hard_inxi.log - flat: yes - - name: run hwinfo shell: hwinfo > hard_hwinfo.log args: chdir: /tmp -- name: retrieve hwinfo log +- name: calculate sha512 checksums of logfiles + stat: + checksum_algorithm: sha512 + get_checksum: true + get_attributes: false + get_md5: false + get_mime: false + path: /tmp/{{item}}.log + register: logfile + with_items: + - hard_dmidecode + - hard_virt_tech + - hard_lscpu + - hard_lsblk + - hard_lshw + - hard_inxi + - hard_hwinfo + +- name: save checksums to files + local_action: + copy content="{{item.stat.checksum}}" dest="../logs/{{inventory_hostname}}/{{item.item}}.log.sha512" + become: no + with_items: + - "{{ logfile.results }}" + +- name: retrieve logs fetch: - src: /tmp/hard_hwinfo.log - dest: ../logs/{{inventory_hostname}}/hard_hwinfo.log + src: /tmp/{{item}}.log + dest: ../logs/{{inventory_hostname}}/{{item}}.log flat: yes + with_items: + - hard_dmidecode + - hard_virt_tech + - hard_lscpu + - hard_lsblk + - hard_lshw + - hard_inxi + - hard_hwinfo - name: remove dmidecode & lswh & inxi & hwinfo apt: @@ -100,11 +97,3 @@ - lshw - inxi - hwinfo - - - - - - - - diff --git a/ansible/roles/lynis/tasks/main.yml b/ansible/roles/lynis/tasks/main.yml index 582b6d6..cc8ec7d 100644 --- a/ansible/roles/lynis/tasks/main.yml +++ b/ansible/roles/lynis/tasks/main.yml @@ -11,29 +11,29 @@ get_url: url: https://github.com/CISOfy/lynis/archive/master.tar.gz dest: /tmp/lynis.tar.gz - when: lynis_log.stat.exists == False + when: lynis_log.stat.exists == False - name: uncompress Lynis unarchive: src: /tmp/lynis.tar.gz dest: /tmp copy: no - when: lynis_log.stat.exists == False + when: lynis_log.stat.exists == False - name: run Lynis command: ./lynis audit system args: chdir: /tmp/lynis-master - when: lynis_log.stat.exists == False + when: lynis_log.stat.exists == False - name: remove Lynis - file: + file: path: "{{item}}" state: absent with_items: - /tmp/lynis-master - /tmp/lynis.tar.gz - when: lynis_log.stat.exists == False + when: lynis_log.stat.exists == False - name: retrieve Lynis log fetch: @@ -47,3 +47,20 @@ script: clean_ips.py lynis.log -p ../logs delegate_to: localhost become: no + +- name: calculate sha512 checksum of logfile + stat: + checksum_algorithm: sha512 + get_checksum: true + get_attributes: false + get_md5: false + get_mime: false + path: ../logs/{{inventory_hostname}}/lynis.log + delegate_to: localhost + become: no + register: logfile + +- name: save checksum to file + local_action: + copy content="{{ logfile.stat.checksum }}" dest="../logs/{{inventory_hostname}}/lynis.log.sha512" + become: no diff --git a/ansible/roles/ports/tasks/main.yml b/ansible/roles/ports/tasks/main.yml index c01799b..10b5a50 100644 --- a/ansible/roles/ports/tasks/main.yml +++ b/ansible/roles/ports/tasks/main.yml @@ -52,6 +52,31 @@ delegate_to: localhost become: no +- name: calculate sha512 checksums of logfiles + stat: + checksum_algorithm: sha512 + get_checksum: true + get_attributes: false + get_md5: false + get_mime: false + path: ../logs/{{inventory_hostname}}/{{item}}.log + register: logfile + delegate_to: localhost + become: no + with_items: + - netstat + - nmap + - nmap_v6 + - nmap_pro + - nmap_pro_v6 + +- name: save checksum to file + local_action: + copy content="{{item.stat.checksum}}" dest="../logs/{{inventory_hostname}}/{{item.item}}.log.sha512" + become: no + with_items: + - "{{ logfile.results }}" + - name: remove nmap apt: name: nmap diff --git a/ansible/roles/speedtest/tasks/main.yml b/ansible/roles/speedtest/tasks/main.yml index bf88e94..5eca378 100644 --- a/ansible/roles/speedtest/tasks/main.yml +++ b/ansible/roles/speedtest/tasks/main.yml @@ -12,7 +12,7 @@ replace: "#!{{ ansible_python_interpreter }}" - name: set execution permission - file: + file: path: /tmp/speedtest-cli mode: "u=rwx,g=rwx,o=rx" @@ -27,19 +27,19 @@ args: chdir: /tmp with_sequence: count=3 - + - name: run speedtest-cli for Barcelona shell: ./speedtest-cli --server 1695 >> speedtest_bar.log args: chdir: /tmp with_sequence: count=3 - + - name: run speedtest-cli for Paris shell: ./speedtest-cli --server 5559 >> speedtest_par.log args: chdir: /tmp with_sequence: count=3 - + - name: run speedtest-cli for London shell: ./speedtest-cli --server 2789 >> speedtest_lon.log args: @@ -57,7 +57,7 @@ args: chdir: /tmp with_sequence: count=3 - + - name: retrieve logs fetch: src: /tmp/{{item}}.log @@ -73,7 +73,7 @@ - speedtest_rom - name: remove speedtest-cli - file: + file: path: "{{item}}" state: absent with_items: @@ -94,3 +94,29 @@ - speedtest_ber - speedtest_rom +- name: calculate sha512 checksums of logfiles + stat: + checksum_algorithm: sha512 + get_checksum: true + get_attributes: false + get_md5: false + get_mime: false + path: ../logs/{{inventory_hostname}}/{{item}}.log + register: logfile + delegate_to: localhost + become: no + with_items: + - speedtest_near + - speedtest_mad + - speedtest_bar + - speedtest_par + - speedtest_lon + - speedtest_ber + - speedtest_rom + +- name: save checksum to file + local_action: + copy content="{{item.stat.checksum}}" dest="../logs/{{inventory_hostname}}/{{item.item}}.log.sha512" + become: no + with_items: + - "{{ logfile.results }}" diff --git a/ansible/roles/sysbench/tasks/main.yml b/ansible/roles/sysbench/tasks/main.yml index 449350e..5feaaae 100644 --- a/ansible/roles/sysbench/tasks/main.yml +++ b/ansible/roles/sysbench/tasks/main.yml @@ -36,12 +36,6 @@ executable: /bin/bash with_sequence: count=5 -- name: retrieve cpu log - fetch: - src: /tmp/sysbench_cpu.log - dest: ../logs/{{inventory_hostname}}/sysbench_cpu.log - flat: yes - - name: run sysbench memory random read shell: sysbench memory --memory-access-mode=rnd --memory-oper=read --num-threads=1 run >> /tmp/sysbench_ram_randrd.log args: @@ -56,15 +50,6 @@ executable: /bin/bash with_sequence: count=5 -- name: retrieve memory logs - fetch: - src: /tmp/{{item}}.log - dest: ../logs/{{inventory_hostname}}/{{item}}.log - flat: yes - with_items: - - sysbench_ram_randrd - - sysbench_ram_randrw - - name: prepare sysbench fileio shell: sysbench fileio --file-total-size=8G prepare args: @@ -89,18 +74,6 @@ args: chdir: /tmp -- name: retrieve randrw log - fetch: - src: /tmp/sysbench_randrw.log - dest: ../logs/{{inventory_hostname}}/sysbench_randrw.log - flat: yes - -- name: retrieve randrd log - fetch: - src: /tmp/sysbench_randrd.log - dest: ../logs/{{inventory_hostname}}/sysbench_randrd.log - flat: yes - - name: set MySQL root password before installing debconf: name: 'mysql-server' @@ -134,11 +107,42 @@ executable: /bin/bash with_sequence: count=5 -- name: retrieve oltp log +- name: calculate sha512 checksums of logfiles + stat: + checksum_algorithm: sha512 + get_checksum: true + get_attributes: false + get_md5: false + get_mime: false + path: /tmp/{{item}}.log + register: logfile + with_items: + - sysbench_ram_randrd + - sysbench_ram_randrw + - sysbench_randrw + - sysbench_randrd + - sysbench_mysql + - sysbench_cpu + +- name: save checksums to files + local_action: + copy content="{{item.stat.checksum}}" dest="../logs/{{inventory_hostname}}/{{item.item}}.log.sha512" + become: no + with_items: + - "{{ logfile.results }}" + +- name: retrieve logs fetch: - src: /tmp/sysbench_mysql.log - dest: ../logs/{{inventory_hostname}}/sysbench_oltp.log + src: /tmp/{{item}}.log + dest: ../logs/{{inventory_hostname}}/{{item}}.log flat: yes + with_items: + - sysbench_ram_randrd + - sysbench_ram_randrw + - sysbench_randrw + - sysbench_randrd + - sysbench_mysql + - sysbench_cpu - name: remove sysbench apt: @@ -147,6 +151,11 @@ purge: yes state: absent +- name: remove sysbench repository + apt_repository: + repo: "deb https://packagecloud.io/akopytov/sysbench/{{ ansible_distribution | lower }}/ {{ ansible_distribution_release }} main" + state: absent + - name: remove mysql apt: name: 'mysql*' diff --git a/ansible/roles/transcode/tasks/main.yml b/ansible/roles/transcode/tasks/main.yml index cfc3719..a7ec45b 100644 --- a/ansible/roles/transcode/tasks/main.yml +++ b/ansible/roles/transcode/tasks/main.yml @@ -30,6 +30,21 @@ chdir: /tmp/ executable: /bin/bash +- name: calculate sha512 checksum of logfile + stat: + checksum_algorithm: sha512 + get_checksum: true + get_attributes: false + get_md5: false + get_mime: false + path: /tmp/transcode.log + register: logfile + +- name: save checksums to files + local_action: + copy content="{{logfile.stat.checksum}}" dest="../logs/{{inventory_hostname}}/transcode.log.sha512" + become: no + - name: retrieve log fetch: src: /tmp/transcode.log diff --git a/ansible/roles/unixbench/tasks/main.yml b/ansible/roles/unixbench/tasks/main.yml index 0f06b98..6d104b3 100644 --- a/ansible/roles/unixbench/tasks/main.yml +++ b/ansible/roles/unixbench/tasks/main.yml @@ -28,13 +28,28 @@ chdir: /tmp/byte-unixbench-master/UnixBench - name: remove UnixBench - file: + file: path: "{{item}}" state: absent with_items: - /tmp/byte-unixbench-master - /tmp/unixbench.tar.gz +- name: calculate sha512 checksum of logfile + stat: + checksum_algorithm: sha512 + get_checksum: true + get_attributes: false + get_md5: false + get_mime: false + path: /tmp/unixbench.log + register: logfile + +- name: save checksums to files + local_action: + copy content="{{logfile.stat.checksum}}" dest="../logs/{{inventory_hostname}}/unixbench.log.sha512" + become: no + - name: retrieve log fetch: src: /tmp/unixbench.log diff --git a/ansible/roles/web/tasks/main.yml b/ansible/roles/web/tasks/main.yml index c86b497..628868c 100644 --- a/ansible/roles/web/tasks/main.yml +++ b/ansible/roles/web/tasks/main.yml @@ -284,3 +284,32 @@ - web_ghost_wrk_100 - web_ghost_wrk_150 - web_ghost_wrk_200 + +- name: calculate sha512 checksums of logfiles + stat: + checksum_algorithm: sha512 + get_checksum: true + get_attributes: false + get_md5: false + get_mime: false + path: ../logs/{{inventory_hostname}}/{{item}}.log + register: logfile + delegate_to: localhost + become: no + with_items: + - web_wp_ab_10 + - web_wp_ab_25 + - web_wp_ab_50 + - web_wp_wrk_100 + - web_wp_wrk_150 + - web_wp_wrk_200 + - web_ghost_wrk_100 + - web_ghost_wrk_150 + - web_ghost_wrk_200 + +- name: save checksum to file + local_action: + copy content="{{item.stat.checksum}}" dest="../logs/{{inventory_hostname}}/{{item.item}}.log.sha512" + become: no + with_items: + - "{{ logfile.results }}" From 0f88ca03ac1dedee002d207326dda1fa08a20b48 Mon Sep 17 00:00:00 2001 From: Christopher Engelhard Date: Tue, 22 May 2018 12:42:47 +0200 Subject: [PATCH 12/12] Rename config files to .template Move hosts and ansible.cfg to hosts.template and ansible.cfg.template, add hosts and ansible.cfg to .gitignore. These files will always have to be changed by the user and therefore should not be part of the repo. --- .gitignore | 3 +++ ansible/{ansible.cfg => ansible.cfg.template} | 0 ansible/{hosts => hosts.template} | 0 3 files changed, 3 insertions(+) create mode 100644 .gitignore rename ansible/{ansible.cfg => ansible.cfg.template} (100%) rename ansible/{hosts => hosts.template} (100%) diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..30914d2 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +*.retry +hosts +ansible.cfg diff --git a/ansible/ansible.cfg b/ansible/ansible.cfg.template similarity index 100% rename from ansible/ansible.cfg rename to ansible/ansible.cfg.template diff --git a/ansible/hosts b/ansible/hosts.template similarity index 100% rename from ansible/hosts rename to ansible/hosts.template