From 41a11242a780e95a1b4fac99d286936be05d8cd6 Mon Sep 17 00:00:00 2001 From: Jared Gillespie Date: Fri, 4 Jan 2019 23:33:17 -0500 Subject: [PATCH 01/21] Removing extraneous scripts --- hoboformat.py | 249 -------------------------------------------- readwritegrouper.py | 141 ------------------------- 2 files changed, 390 deletions(-) delete mode 100644 hoboformat.py delete mode 100644 readwritegrouper.py diff --git a/hoboformat.py b/hoboformat.py deleted file mode 100644 index 03cfa91..0000000 --- a/hoboformat.py +++ /dev/null @@ -1,249 +0,0 @@ -#!/usr/bin/python3 -# Utility for utilizing HOBOware files w/ iobs tool. -# Copyright (c) 2018, UofL Computer Systems Lab. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without event the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -__author__ = 'Jared Gillespie' -__version__ = '0.1.0' - -from time import mktime, strptime, struct_time - -import os -import sys - - -class RowInfo: - """A single row for a CSV.""" - - def __init__(self, line, start_time, stop_time, io_kbytes): - self.line = line - self.joules = 0 - self.start_time = start_time - self.stop_time = stop_time - self.io_kbytes = io_kbytes - - def __str__(self): - if self.io_kbytes == 0: - return '%s,%0.2f' % (self.line, self.joules) - else: - return '%s,%0.2f,%0.2f' % (self.line, self.joules, self.io_kbytes / self.joules) - - -def search_single(hobo_file: str, start_time: struct_time, stop_time: struct_time): - """Averages values in a HOBO file between a range. - - :param hobo_file: The HOBO file to search. - :param start_time: The inclusive start of the average range. - :param stop_time: The inclusive stop of the average range. - """ - joules = 0 - lc = 0 - found = False - with open(hobo_file, 'r') as file: - for line in file: - lc += 1 - - # Skip first two lines (title + header) - if lc < 3: - continue - - _, date_time, _, _, w, _, _, _ = line.split(',') - - date_time = strptime(date_time, '%m/%d/%y %I:%M:%S %p') - - if start_time <= date_time <= stop_time: - found = True - joules += float(w) - else: - if start_time > date_time: - continue - - # Must have past stop time - if not found: - print('Unable to find values within specified range!') - usage() - sys.exit(1) - else: - break - - if not found: - print('Unable to find values within specified range!') - usage() - sys.exit(1) - - print('Joules: %0.2f' % joules) - print('Timespan: %ss' % int(mktime(stop_time) - mktime(start_time) + 1)) - - -def search_csv(hobo_file: str, inp_file: str): - """Calculates joules in a HOBO file for each row in the input file. - - :param hobo_file: - :param inp_file: - """ - row_infos = [] - header = '' - start_index, stop_index, io_kbytes_index = -1, -1, -1 - lc = 0 - - # Read in input file - with open(inp_file, 'r') as file: - for line in file: - lc += 1 - - # Find column indexes of start and stop - if lc == 1: - header = line.strip() - start_index, stop_index, io_kbytes_index = search_header(line.strip()) - continue - - split = line.strip().split(',') - start_time = strptime(split[start_index], '%m/%d/%y %I:%M:%S %p') - stop_time = strptime(split[stop_index], '%m/%d/%y %I:%M:%S %p') - if io_kbytes_index != -1: - io_kbytes = float(split[io_kbytes_index]) - else: - io_kbytes = 0 - row_infos.append(RowInfo(line.strip(), start_time, stop_time, io_kbytes)) - - lc = 0 - - # Read in HOBO file - with open(hobo_file, 'r') as file: - for line in file: - lc += 1 - - # Skip first two lines (title + header) - if lc < 3: - continue - - try: - _, date_time, _, _, w, _, _, _ = line.strip().split(',') - except: - try: - _, date_time, _, _, w, _, _ = line.strip().split(',') - except: - continue - - date_time = strptime(date_time, '%m/%d/%y %I:%M:%S %p') - - for row_info in row_infos: - if row_info.start_time <= date_time <= row_info.stop_time: - row_info.joules += float(w) - - # Write output file - with open(inp_file, 'w') as file: - file.write(header + ',joules') - if io_kbytes_index == -1: - file.write('\n') - else: - file.write(',kbpj\n') - - for row_info in row_infos: - file.write(str(row_info)) - file.write('\n') - - -def search_header(header: str): - """Parses header for "start-time" and "stop-time" indexes. - - :param header: The header to parse. - :return: A tuple containing the (start_index, stop_index, io_kbytes_index). - """ - start_index, stop_index, io_kbytes_index = -1, -1, -1 - - for index, column in enumerate(header.split(',')): - if column == 'start-time': - start_index = index - elif column == 'stop-time': - stop_index = index - elif column == 'io-kbytes': - io_kbytes_index = index - - if start_index == -1: - raise Exception('Unable to parse header, expected "start-time"!') - - if stop_index == -1: - raise Exception('Unable to parse header, expected "stop-time"!') - - return start_index, stop_index, io_kbytes_index - - -def usage(): - """Displays command-line information.""" - name = os.path.basename(__file__) - print('%s %s' % (name, __version__)) - print('Usage: %s ' % name) - print(' %s ' % name) - print(' : Is expected to be a hobo output file with a title, a header, and rows with the following') - print(' : attributes: #, Date Time, V, A, W, Wh, Stopped, End Of File.') - print(' : Is the inclusive start of the range of values. Should be of the form MM/DD/YY HH:MM:SS PP.') - print(' : Is the inclusive stop of the range of values. Should be of the form MM/DD/YY HH:MM:SS PP.') - print(' : Is the input file to append columns to. Expects a header, command delimited, and at least') - print(' : two columns: start-time and stop-time.') - print('Output: If and are given, the average V, A, W, and Wh are given.') - print(' Else, if is given, the columns are added to each row.') - - -def main(argv: list): - if '-h' in argv or '--help' in argv: - usage() - sys.exit(1) - - if len(argv) == 2: - hobo_file, inp_file = argv - - if not os.path.isfile(hobo_file): - print('HOBO file given does not exist: %s' % hobo_file) - usage() - sys.exit(1) - - if not os.path.isfile(inp_file): - print('Input file given does not exist: %s' % inp_file) - usage() - sys.exit(1) - - search_csv(hobo_file, inp_file) - elif len(argv) == 3: - hobo_file, start_time, stop_time = argv - - if not os.path.isfile(hobo_file): - print('HOBO file given does not exist: %s' % hobo_file) - usage() - sys.exit(1) - - try: - start_time = strptime(start_time, '%m/%d/%y %I:%M:%S %p') - except: - print('Unable to parse given start time: %s' % start_time) - usage() - sys.exit(1) - - try: - stop_time = strptime(stop_time, '%m/%d/%y %I:%M:%S %p') - except: - print('Unable to parse given stop time: %s' % stop_time) - usage() - sys.exit(1) - - search_single(hobo_file, start_time, stop_time) - else: - usage() - sys.exit(1) - - -if __name__ == '__main__': - main(sys.argv[1:]) diff --git a/readwritegrouper.py b/readwritegrouper.py deleted file mode 100644 index 3c34202..0000000 --- a/readwritegrouper.py +++ /dev/null @@ -1,141 +0,0 @@ -#!/usr/bin/python3 -# Utility for utilizing grouping iobs output workloads of read / write only into single columns. -# Copyright (c) 2018, UofL Computer Systems Lab. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without event the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -__author__ = 'Jared Gillespie' -__version__ = '0.1.0' - -import os -import sys - - -def aggregate_csv(inp_file: str): - """Calculates joules in a HOBO file for each row in the input file. - - :param inp_file: - """ - new_header = '' - new_lines = [] - merged_indices = [] - lc = 0 - - # Read in input file - with open(inp_file, 'r') as file: - for line in file: - new_line = [] - lc += 1 - - # Find column indexes of start and stop - if lc == 1: - new_header, merged_indices = search_header(line.strip()) - continue - - split = line.strip().split(',') - - is_read = True - first_pair = True - last_index = -1 - - for read_index, write_index in merged_indices: - if first_pair: - first_pair = False - is_read = float(split[read_index]) != 0 - - # Add prior indices - for index in range(last_index + 1, read_index): - new_line.append(split[index]) - - last_index = write_index - - if is_read: - new_line.append(split[read_index]) - else: - new_line.append(split[write_index]) - - for index in range(last_index + 1, len(split)): - new_line.append(split[index]) - - new_lines.append(new_line) - - # Write output file - with open(inp_file, 'w') as file: - file.write(new_header + '\n') - - for line in new_lines: - file.write(','.join(map(str, line)) + '\n') - - -def search_header(header: str): - """Parses header columns ending in "-read" or "-write", yielding new header format and merged indices. - - Note this assumes if a column ending in "-read" exists, then a column ending in "-write" also exists right after it. - - :param header: The header to parse. - :return: A tuple containing the (new header, a list of tuple pairs of indices to merge). - ex. ('device,workload,throughput', (2, 3)) - """ - new_header = [] - merged_indices = [] - - split = header.split(',') - index = 0 - - while index < len(split): - if '-read' in split[index]: - new_header.append(split[index].replace('-read', '')) - merged_indices.append((index, index + 1)) - index += 1 - else: - new_header.append(split[index]) - - index += 1 - - return ','.join(new_header), merged_indices - - -def usage(): - """Displays command-line information.""" - name = os.path.basename(__file__) - print('%s %s' % (name, __version__)) - print('Usage: %s ' % name) - print('Command Line Arguments:') - print(' : The iobs output file to modify.') - print('Output: Aggregates read / write columns (such as throughput-read, throughput-write, etc.) into a single column.') - - -def main(argv: list): - if '-h' in argv or '--help' in argv: - usage() - sys.exit(1) - - if len(argv) != 1: - usage() - sys.exit(1) - - inp_file = argv[0] - - if not os.path.isfile(inp_file): - print('Input file given does not exist: %s' % inp_file) - usage() - sys.exit(1) - - aggregate_csv(inp_file) - - -if __name__ == '__main__': - main(sys.argv[1:]) From 4e560a060bc81c309f501c7a90ccabe87aae4d8f Mon Sep 17 00:00:00 2001 From: Jared Gillespie Date: Sat, 5 Jan 2019 17:31:08 -0500 Subject: [PATCH 02/21] Adding changelog --- CHANGELOG.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..276556a --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,13 @@ +Changelog +========= +All notable changes to this project will be documented in this file. + +The format is based on `Keep a Changelog`_ and this project adheres to `Semantic Versioning`_. + +.. _Keep a Changelog: http://keepachangelog.com/en/1.0.0/ +.. _Semantic Versioning: http://semver.org/spec/v2.0.0.html + +`Unreleased`_ +------------- + +.. _Unreleased: https://github.com/uofl-csl/proxyscrape/compare/v1.0.0...HEAD From 0ef3c71a8ce1676fc492d5624d4a0fc6f016b7ff Mon Sep 17 00:00:00 2001 From: Jared Gillespie Date: Thu, 14 Feb 2019 19:53:32 -0500 Subject: [PATCH 03/21] Cleaning gitignore --- .gitignore | 41 ++--------------------------------------- 1 file changed, 2 insertions(+), 39 deletions(-) diff --git a/.gitignore b/.gitignore index 7bbc71c..1bb90c6 100644 --- a/.gitignore +++ b/.gitignore @@ -46,20 +46,8 @@ coverage.xml *.cover .hypothesis/ -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy +# IDE +.idea/ # Sphinx documentation docs/_build/ @@ -67,18 +55,6 @@ docs/_build/ # PyBuilder target/ -# Jupyter Notebook -.ipynb_checkpoints - -# pyenv -.python-version - -# celery beat schedule file -celerybeat-schedule - -# SageMath parsed files -*.sage.py - # dotenv .env @@ -86,16 +62,3 @@ celerybeat-schedule .venv venv/ ENV/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ From 9669eba863cc08f0c5c54e4344f28aad242dfe34 Mon Sep 17 00:00:00 2001 From: Jared Gillespie Date: Fri, 15 Feb 2019 18:11:23 -0500 Subject: [PATCH 04/21] Removing examples --- examples/fio-rr-rw-sr-sw.iobs | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 examples/fio-rr-rw-sr-sw.iobs diff --git a/examples/fio-rr-rw-sr-sw.iobs b/examples/fio-rr-rw-sr-sw.iobs deleted file mode 100644 index f4fced9..0000000 --- a/examples/fio-rr-rw-sr-sw.iobs +++ /dev/null @@ -1,19 +0,0 @@ -[global] -device=/dev/sda -repetition=3 -runtime=10 -schedulers=cfq,noop,deadline -delay=1 -workload=fio - -[fio-sequential-read] -command=fio --name=fio-seq-read --rw=read --bs=1024K --direct=1 --numjobs=1 --time_based=1 --runtime=30 --filename=fiofile --size=10G --ioengine=libaio --iodepth=1 - -[fio-sequential-write] -command=fio --name=fio-seq-write --rw=write --bs=1024K --direct=1 --numjobs=1 --time_based=1 --runtime=30 --filename=fiofile --size=10G --ioengine=libaio --iodepth=1 - -[fio-random-read] -command=fio --name=fio-rand-read --rw=randread --bs=1024K --direct=1 --numjobs=1 --time_based=1 --runtime=30 --filename=fiofile --size=10G --ioengine=libaio --iodepth=1 - -[fio-random-write] -command=fio --name=fio-rand-write --rw=randwrite --bs=1024K --direct=1 --numjobs=1 --time_based=1 --runtime=30 --filename=fiofile --size=10G --ioengine=libaio --iodepth=1 \ No newline at end of file From 98d4c1318cb9b8d694583d8a5af44e711be1196a Mon Sep 17 00:00:00 2001 From: Jared Gillespie Date: Fri, 15 Feb 2019 18:12:05 -0500 Subject: [PATCH 05/21] Setting up as package --- AUTHORS | 4 ++++ MANIFEST.in | 6 +++++ iobs/__init__.py | 27 +++++++++++++++++++++ setup.cfg | 5 ++++ setup.py | 62 ++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 104 insertions(+) create mode 100644 AUTHORS create mode 100644 MANIFEST.in create mode 100644 iobs/__init__.py create mode 100644 setup.cfg create mode 100644 setup.py diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000..d8e9cf6 --- /dev/null +++ b/AUTHORS @@ -0,0 +1,4 @@ +# A list of people who have contributed to iobs. + +Jared Gillespie +Martin Heil diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..165ece1 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,6 @@ +include README.md +include CHANGELOG.md +include LICENSE +include AUTHORS + +recursive-include examples diff --git a/iobs/__init__.py b/iobs/__init__.py new file mode 100644 index 0000000..5803ef8 --- /dev/null +++ b/iobs/__init__.py @@ -0,0 +1,27 @@ +# Copyright (c) 2018, UofL Computer Systems Lab. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without event the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +__title__ = 'iobs' +__summary__ = 'Linux I/O Benchmark for Schedulers' +__url__ = 'https://github.com/uofl-csl/iobs' + +__version__ = '0.3.1' + +__author__ = 'Jared Gillespie, Martin Heil' +__email__ = 'jared.gillespie@louisville.edu' + +__license__ = 'GNU General Public License v2 (GPLv2)' +__copyright__ = 'Copyright (c) 2018, UofL Computer Systems Lab.' diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..ed8a958 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,5 @@ +[bdist_wheel] +universal = 1 + +[metadata] +license_file = LICENSE diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..827a694 --- /dev/null +++ b/setup.py @@ -0,0 +1,62 @@ +# Copyright (c) 2018, UofL Computer Systems Lab. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without event the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from setuptools import setup + +import iobs + + +setup( + name=iobs.__title__, + version=iobs.__version__, + description=iobs.__summary__, + long_description=open('README.md').read(), + url=iobs.__url__, + project_urls={ + 'UofL CSL': 'http://cecs.louisville.edu/csl/', + 'IOBS source': 'https://github.com/uofl-csl/iobs' + }, + + author=iobs.__author__, + author_email=iobs.__email__, + license='GNU GPLv2', + classifiers=[ + 'Intended Audience :: Developers', + 'License :: OSI Approved :: GNU General Public License v2 (GPLv2)', + 'Natural Language :: English', + 'Operating System :: POSIX :: Linux', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: Implementation :: CPython', + ], + packages=['iobs', 'iobs.commands'], + include_package_data=True, + + entry_points={ + 'iobs.registered_commands': [ + 'execute = iobs.commands.execute:main' + ], + 'console_scripts': [ + 'iobs = iobs.__main__:main' + ] + }, + + python_requires='>=3.4', + install_requires=[] +) From 0b92841bfd4393d18dc9b5521d95ad8805627d54 Mon Sep 17 00:00:00 2001 From: Jared Gillespie Date: Mon, 25 Feb 2019 14:44:57 -0500 Subject: [PATCH 06/21] Adding cli --- iobs/__init__.py | 5 +- iobs/__main__.py | 62 +++++++++++++++++++++++++ iobs/cli.py | 96 +++++++++++++++++++++++++++++++++++++++ iobs/commands/__init__.py | 16 +++++++ setup.py | 4 +- 5 files changed, 180 insertions(+), 3 deletions(-) create mode 100644 iobs/__main__.py create mode 100644 iobs/cli.py create mode 100644 iobs/commands/__init__.py diff --git a/iobs/__init__.py b/iobs/__init__.py index 5803ef8..12502fe 100644 --- a/iobs/__init__.py +++ b/iobs/__init__.py @@ -14,13 +14,14 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + __title__ = 'iobs' __summary__ = 'Linux I/O Benchmark for Schedulers' __url__ = 'https://github.com/uofl-csl/iobs' -__version__ = '0.3.1' +__version__ = '1.0.0' -__author__ = 'Jared Gillespie, Martin Heil' +__author__ = 'Jared Gillespie' __email__ = 'jared.gillespie@louisville.edu' __license__ = 'GNU General Public License v2 (GPLv2)' diff --git a/iobs/__main__.py b/iobs/__main__.py new file mode 100644 index 0000000..3892f99 --- /dev/null +++ b/iobs/__main__.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python +# Copyright (c) 2018, UofL Computer Systems Lab. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without event the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import colorama +import signal +import sys + +from iobs.cli import dispatch +from iobs.errors import IOBSBaseException +from iobs.output import printf, PrintType +from iobs.process import ProcessManager + + +def sig_handler(signal, frame): + """Signal handler for termination signals sent to main process. + + Args: + signal: The signal. + frame: The frame. + """ + printf('Program encountered termination signal, aborting...', + print_type=PrintType.ERROR | PrintType.ERROR_LOG) + + ProcessManager.kill_processes() + ProcessManager.clear_processes() + + colorama.deinit() + + sys.exit(1) + + +def main(): + try: + return dispatch(sys.argv[1:]) + except IOBSBaseException as err: + printf('Program encountered critical error\n{}'.format(err), + print_type=PrintType.ERROR | PrintType.ERROR_LOG) + return '{}: {}'.format(err.__class__.__name__, err.args[0]) + + +if __name__ == '__main__': + signal.signal(signal.SIGTERM, sig_handler) # Process termination + signal.signal(signal.SIGINT, sig_handler) # Keyboard interrupt + + colorama.init() + + sys.exit(main()) diff --git a/iobs/cli.py b/iobs/cli.py new file mode 100644 index 0000000..494cefc --- /dev/null +++ b/iobs/cli.py @@ -0,0 +1,96 @@ +# Copyright (c) 2018, UofL Computer Systems Lab. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without event the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import argparse +import pkg_resources +import setuptools + +import colorama + +import iobs + + +def _registered_commands(group='iobs.registered_commands'): + """Retrieves registered commands for a entry point group. + + Args: + group: The group. + + Returns: + A dictionary mapping entry point name to entry point for each entry + in the group. + """ + return {c.name: c for c in pkg_resources.iter_entry_points(group=group)} + + +def list_dependencies_and_versions(): + """Retrieves a list of package dependencies and their current versions. + + Returns: + List of tuples containing package dependency name and version. + """ + return [ + ('colorama', colorama.__version__), + ('setuptools', setuptools.__version__) + ] + + +def dep_versions(): + """Retrieves string of package dependencies. + + Returns: + String of package dependencies and their current versions. + """ + return ', '.join( + '{}: {}'.format(*dependency) + for dependency in list_dependencies_and_versions() + ) + + +def dispatch(argv): + """Dispatches execution to the appropriate command. + + Args: + argv: The command-line arguments. + + Returns: + Execution of the command. + """ + registered_commands = _registered_commands() + parser = argparse.ArgumentParser(prog='iobs') + parser.add_argument( + '--version', + action='version', + version='%(prog)s versions {} ({})'.format( + iobs.__version__, + dep_versions() + ) + ) + parser.add_argument( + 'command', + choices=registered_commands.keys() + ) + parser.add_argument( + 'args', + help=argparse.SUPPRESS, + nargs=argparse.REMAINDER + ) + + args = parser.parse_args(argv) + + main = registered_commands[args.command].load() + return main(args.args) diff --git a/iobs/commands/__init__.py b/iobs/commands/__init__.py new file mode 100644 index 0000000..6e947c6 --- /dev/null +++ b/iobs/commands/__init__.py @@ -0,0 +1,16 @@ +#!/usr/bin/python3 +# Copyright (c) 2018, UofL Computer Systems Lab. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without event the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. diff --git a/setup.py b/setup.py index 827a694..ceef1b9 100644 --- a/setup.py +++ b/setup.py @@ -58,5 +58,7 @@ }, python_requires='>=3.4', - install_requires=[] + install_requires=[ + 'colorama' + ] ) From e23c45bd5a3a0fdc0e90b092f8b502bdd0b74e79 Mon Sep 17 00:00:00 2001 From: Jared Gillespie Date: Mon, 25 Feb 2019 15:11:51 -0500 Subject: [PATCH 07/21] Splitting up iobs.py --- iobs.py | 1664 -------------------------------------- iobs/commands/execute.py | 189 +++++ iobs/config.py | 1115 +++++++++++++++++++++++++ iobs/errors.py | 87 ++ iobs/output.py | 68 ++ iobs/process.py | 485 +++++++++++ iobs/settings.py | 255 ++++++ iobs/util.py | 84 ++ 8 files changed, 2283 insertions(+), 1664 deletions(-) delete mode 100644 iobs.py create mode 100644 iobs/commands/execute.py create mode 100644 iobs/config.py create mode 100644 iobs/errors.py create mode 100644 iobs/output.py create mode 100644 iobs/process.py create mode 100644 iobs/settings.py create mode 100644 iobs/util.py diff --git a/iobs.py b/iobs.py deleted file mode 100644 index 02a6c51..0000000 --- a/iobs.py +++ /dev/null @@ -1,1664 +0,0 @@ -#!/usr/bin/python3 -# Linux I/O Benchmark for Schedulers -# Copyright (c) 2018, UofL Computer Systems Lab. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without event the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -__author__ = 'Jared Gillespie, Martin Heil' -__version__ = '0.3.1' - - -from collections import defaultdict -from functools import wraps -from getopt import getopt, GetoptError -from time import strftime, localtime - -import configparser -import json -import logging -import os -import platform -import multiprocessing -import re -import stat -import shlex -import signal -import subprocess -import sys -import time - - -# region termination -def sig_handler(signal, frame): - if Mem.current_processes: - kill_processes(Mem.current_processes) - Mem.current_processes.clear() - - sys.exit(0) -# endregion - - -# region utils -def adjusted_workload(command: str, workload: str): - """Adjusts a command by adding extra flags, etc. - - :param command: The command. - :param workload: The workload. - :return: The adjusted workload command. - """ - if workload == 'fio': - return '%s %s' % (command, '--output-format=json') - - return command - - -def ignore_exception(exception=Exception, default_val=None, should_log=True): - """A decorator function that ignores the exception raised, and instead returns a default value. - - :param exception: The exception to catch. - :param default_val: The default value. - :param should_log: Whether the exception should be logged. - :return: The decorated function. - """ - def decorator(func): - @wraps(func) - def wrapper(*args, **kwargs): - try: - return func(*args, **kwargs) - except exception as e: - log(str(e)) - return default_val - return wrapper - return decorator - - -def log_around(before_message: str=None, after_message: str=None, exception_message: str=None, ret_validity: bool=False): - """Logs messages around a function. - - :param before_message: The message to log before. - :param after_message: The message to log after. - :param exception_message: The message to log when an exception occurs. - :param ret_validity: If true, if the function returns False or None, the exception message is printed. - :return: The decorated function. - """ - def decorator(func): - @wraps(func) - def wrapper(*args, **kwargs): - if before_message: - log(before_message) - - try: - out = func(*args, **kwargs) - - if ret_validity: - if out is False or out is None: - if exception_message: - log(exception_message) - return out - elif after_message: - log(after_message) - elif after_message: - log(after_message) - - return out - except Exception as e: - log(e) - if exception_message: - log(exception_message) - raise - return wrapper - return decorator - - -def log(*args, **kwargs): - """Logs a message if logging is enabled. - - :param args: The arguments. - :param kwargs: The keyword arguments. - """ - if Mem.log: - if args: - args_rem = [a.strip() if isinstance(a, str) else a for a in args][1:] - message = args[0] - - for line in str(message).split('\n'): - logging.debug(line, *args_rem, **kwargs) - else: - logging.debug(*args, **kwargs) - - -def print_and_log(*args, **kwargs): - """Prints a message, and logs if logging is enabled. - - :param args: The arguments. - :param kwargs: The keyword arguments. - """ - log(*args, **kwargs) - args = [a.strip() if isinstance(a, str) else a for a in args] - print(*args, **kwargs) - - -def print_detailed(*args, **kwargs): - """Prints a message if verbose is enabled, and logs if logging is enabled. - - :param args: The arguments. - :param kwargs: The keyword arguments. - """ - log(*args, **kwargs) - print_verbose(*args, **kwargs) - - -def print_verbose(*args, **kwargs): - """Prints a message if verbose is enabled. - - :param args: The arguments. - :param kwargs: The keyword arguments. - """ - if Mem.verbose: - args = [a.strip() if isinstance(a, str) else a for a in args] - print(*args, **kwargs) - - -def try_split(s: str, delimiter) -> list: - """Tries to split a string by the given delimiter(s). - - :param s: The string to split. - :param delimiter: Either a single string, or a tuple of strings (i.e. (',', ';'). - :return: Returns the string split into a list. - """ - if isinstance(delimiter, tuple): - for d in delimiter: - if d in s: - return [i.strip() for i in s.split(d)] - elif delimiter in s: - return s.split(delimiter) - - return [s] -# endregion - - -# region classes -class Mem: - """A simple data-store for persisting and keeping track of global data.""" - - def __init__(self): - # Constants - self.GLOBAL_HEADER = 'global' - - # Settings - self.cleanup = False - self.config_file = None - self.continue_on_failure = False - self.jobs = [] - self.log = False - self.output_file = None - self.retry = 1 - self.verbose = False - - # Global Job Settings - self._command = None - self._delay = 0 - self._device = None - self._repetition = 1 - self._runtime = None - self._schedulers = None - self._workload = None - - # Formatters - self.format_blktrace = 'blktrace -d %s -o %s -w %s -b 16384 -n 8' # device, file prefix, runtime - self.format_blkparse = "blkparse -i %s -d %s.blkparse.bin -q -O -M" # file prefix, file prefix - self.format_btt = 'btt -i %s.blkparse.bin' # file prefix - self.format_blkrawverify = 'blkrawverify %s' # file prefix - - # Regex - self.re_blkparse_throughput_read = re.compile(r'Throughput \(R/W\): (\d+)[a-zA-Z]+/s') - self.re_blkparse_throughput_write = re.compile(r'Throughput \(R/W\): (?:\d+)[a-zA-Z]+/s / (\d+)[a-zA-z]+/s') - self.re_btt_d2c = re.compile(r'D2C\s*(?:\d+.\d+)\s*(\d+.\d+)\s*(?:\d+.\d+)\s*(?:\d+)') - self.re_btt_q2c = re.compile(r'Q2C\s*(?:\d+.\d+)\s*(\d+.\d+)\s*(?:\d+.\d+)\s*(?:\d+)') - self.re_device = re.compile(r'/dev/(.*)') - - # Validity - self.valid_global_settings = {'command', 'delay', 'device', 'schedulers', 'repetition', 'runtime', 'workload'} - self.valid_job_settings = {'command', 'delay', 'device', 'schedulers', 'repetition', 'runtime', 'workload'} - self.valid_workloads = {'fio'} - - # Other - self.current_processes = set() # Keep track of current processes for killing purposes - self.output_column_order = ['device', - 'io-depth', - 'workload', - 'scheduler', - 'slat-read', 'slat-write', - 'clat-read', 'clat-write', - 'lat-read', 'lat-write', - 'q2c', - 'd2c', - 'fslat-read', 'fslat-write', - 'bslat', - 'iops-read', 'iops-write', - 'throughput-read', 'throughput-write', - 'io-kbytes', - 'start-time', 'stop-time'] - - @property - def command(self) -> str: - return self._command - - @command.setter - def command(self, value: str): - self._command = value - - @property - def delay(self) -> int: - return self._delay - - @delay.setter - def delay(self, value: int): - conv_value = ignore_exception(ValueError, -1)(int)(value) - - if conv_value < 1: - raise ValueError('Delay given is < 0: %s' % value) - - self._delay = conv_value - - @property - def device(self) -> str: - return self._device - - @device.setter - def device(self, value: str): - self._device = value - - @property - def repetition(self) -> int: - return self._repetition - - @repetition.setter - def repetition(self, value: int): - conv_value = ignore_exception(ValueError, 0)(int)(value) - - if conv_value < 1: - raise ValueError('Repetition given is < 1: %s' % value) - - self._repetition = conv_value - - @property - def runtime(self): - return self._runtime - - @runtime.setter - def runtime(self, value: int): - conv_value = ignore_exception(ValueError, 0)(int)(value) - - if conv_value < 1: - raise ValueError('Runtime given is < 1: %s' % value) - - self._runtime = conv_value - - @property - def schedulers(self) -> set: - return self._schedulers - - @schedulers.setter - def schedulers(self, value): - self._schedulers = set(try_split(value, ',')) - - @property - def workload(self) -> str: - return self._workload - - @workload.setter - def workload(self, value: str): - self._workload = value - - @log_around('Processing jobs', 'Processed jobs successfully', 'Failed to process all jobs', True) - def process_jobs(self) -> bool: - """Executes each job. - - :return: Returns True if successful, else False. - """ - for job in self.jobs: - if not job.execute(): - if not self.continue_on_failure: - return False - - return True - - -# Turns the class into a singleton (this is some sneaky stuff) -Mem = Mem() - - -class Job: - """A single job, which is representative of a single workload to be run.""" - - def __init__(self, name: str): - self._name = name - self._command = None - self._delay = None - self._device = None - self._repetition = None - self._runtime = None - self._schedulers = None - self._workload = None - - @property - def name(self) -> str: - return self._name - - @property - def command(self) -> str: - return self._command - - @command.setter - def command(self, value: str): - self._command = value - - @property - def delay(self) -> int: - return self._delay - - @delay.setter - def delay(self, value: int): - conv_value = ignore_exception(ValueError, -1)(int)(value) - - if conv_value < 1: - raise ValueError('Delay given is < 0: %s' % value) - - self._delay = conv_value - - @property - def device(self) -> str: - return self._device - - @device.setter - def device(self, value): - self._device = value - - @property - def repetition(self) -> int: - return self._repetition - - @repetition.setter - def repetition(self, value: int): - conv_value = ignore_exception(ValueError, 0)(int)(value) - - if conv_value < 1: - raise ValueError('Repetition given is < 1: %s' % value) - - self._repetition = conv_value - - @property - def runtime(self) -> int: - return self._runtime - - @runtime.setter - def runtime(self, value: int): - conv_value = ignore_exception(ValueError, 0)(int)(value) - - if conv_value < 1: - raise ValueError('Runtime given is < 1: %s' % value) - - self._runtime = conv_value - - @property - def schedulers(self) -> set: - return self._schedulers - - @schedulers.setter - def schedulers(self, value): - self._schedulers = set(try_split(value, ',')) - - @property - def workload(self) -> str: - return self._workload - - @workload.setter - def workload(self, value): - self._workload = value - - @log_around(after_message='Job executed successfully', exception_message='Job failed', ret_validity=True) - def execute(self) -> bool: - """Executes a single job. - - :return: Returns True if successful, else False. - """ - log('Executing job %s' % self.name) - - metrics_store = MetricsStore() - - for scheduler in self.schedulers: - - if not change_scheduler(scheduler, self.device): - print_detailed('Unable to change scheduler %s for device %s' % (scheduler, self.device)) - return False - - start_time, stop_time, metrics = self._execute_workload() - - metrics = defaultdict(int, metrics) - metrics['fslat-read'] = metrics['clat-read'] - metrics['q2c'] - metrics['fslat-write'] = metrics['clat-write'] - metrics['q2c'] - metrics['bslat'] = metrics['q2c'] - metrics['d2c'] - metrics['workload'] = self.name - metrics['device'] = self.device - metrics['scheduler'] = scheduler - metrics['start-time'] = start_time - metrics['stop-time'] = stop_time - - Metrics.print(self.name, self.workload, scheduler, self.device, metrics) - Metrics.output(metrics) - - device_short = Mem.re_device.findall(self.device)[0] - metrics_store.add(self.workload, device_short, scheduler, metrics) - - return True - - def fill_missing(self, o): - """Fills in missing values from object. - - :param o: The object. - """ - if self._delay is None: - self._delay = ignore_exception(AttributeError)(getattr)(o, 'delay') - - if self._device is None: - self._device = ignore_exception(AttributeError)(getattr)(o, 'device') - - if self._repetition is None: - self._repetition = ignore_exception(AttributeError)(getattr)(o, 'repetition') - - if self._runtime is None: - self._runtime = ignore_exception(AttributeError)(getattr)(o, 'runtime') - - if self._schedulers is None: - self._schedulers = ignore_exception(AttributeError)(getattr)(o, 'schedulers') - - if self._workload is None: - self._workload = ignore_exception(AttributeError)(getattr)(o, 'workload') - - def get_invalid_props(self) -> list: - """Returns the properties that are invalid. - - :return: A list of properties. - """ - invalid_props = [] - - if self._delay is None: - invalid_props.append('delay') - - if self._device is None: - invalid_props.append('device') - - if self._repetition is None: - invalid_props.append('repetition') - - if self._runtime is None: - invalid_props.append('runtime') - - if self._schedulers is None: - invalid_props.append('schedulers') - - if self._workload is None: - invalid_props.append('workload') - - return invalid_props - - def is_valid(self) -> bool: - """Returns whether the job is valid. - - :return: Returns True if valid, else False. - """ - return self._delay is not None and \ - self._device is not None and \ - self._repetition is not None and \ - self._runtime is not None and \ - self._schedulers is not None and \ - self._workload is not None - - def _execute_workload(self) -> tuple: - """Executes a workload. - - :return: Returns a dictionary of metrics if successful, else None. - """ - log('Executing workload %s' % self.workload) - - metrics = Metrics(self.workload) - - start_time = '' - stop_time = '' - - # Repeat job multiple times - for i in range(self.repetition): - device_short = Mem.re_device.findall(self.device)[0] - - # Repeat workload if failure - retry = 0 - while retry < Mem.retry: - retry += 1 - - # Clear all the things - clear_caches(self.device) - - # Run workload along with blktrace - blktrace = Mem.format_blktrace % (self.device, device_short, self.runtime) - - adj_command = adjusted_workload(self.command, self.workload) - - start_time = strftime('%m/%d/%y %I:%M:%S %p', localtime()) - out = run_parallel_commands([('blktrace', self.delay, blktrace), (self.workload, 0, adj_command)]) - stop_time = strftime('%m/%d/%y %I:%M:%S %p', localtime()) - - # Error running commands - if out is None or 'blktrace' in out and out['blktrace'] is None: - log('Error running workload %s' % self.workload) - time.sleep(5) - continue - - blktrace_out, _ = out['blktrace'] - workload_out, _ = out[self.workload] - - log('Workload Output') - log(workload_out) - - if blktrace_out is None or workload_out is None: - log('Error running workload %s' % self.workload) - time.sleep(5) - continue - - # Run blkparse - blkparse = Mem.format_blkparse % (device_short, device_short) - - _, _ = run_command(blkparse, ignore_output=True) - - # Way too much cowbell (-f '' doesn't seem to trim output) - #log('BLKPARSE Output') - #log(blkparse_out) - - # Run blkrawverify - blkrawverify = Mem.format_blkrawverify % device_short - - blkrawverify_out, _ = run_command(blkrawverify) - - log('BLKRAWYVERIFY Output') - log(blkrawverify_out) - - # Run btt - btt = Mem.format_btt % device_short - - btt_out, _ = run_command(btt) - - if btt_out is None: - log('Error running workload %s' % self.workload) - time.sleep(5) - continue - - log('BTT Output') - btt_split = btt_out.split("# Total System")[0] - btt_split2 = btt_split.split("==================== All Devices ====================")[-1] - log("==================== All Devices ====================") - log(btt_split2) - - # Cleanup intermediate files - if Mem.cleanup: - log('Cleaning up files') - cleanup_files('sda.blktrace.*', 'sda.blkparse.*', 'sys_iops_fp.dat', 'sys_mbps_fp.dat') - - dmm = get_device_major_minor(self.device) - cleanup_files('%s_iops_fp.dat' % dmm, '%s_mbps_fp.dat' % dmm) - cleanup_files('%s.verify.out' % device_short) - - m = Metrics.gather_metrics(None, btt_out, workload_out, self.workload) - metrics.add_metrics(m) - - break - else: - print_detailed('Unable to run workload %s' % self.workload) - return None - - return start_time, stop_time, metrics.average_metrics() - - -class MetricsStore: - """A datastore for saving / retrieving metrics.""" - - def __init__(self): - self._store = dict() - - def __contains__(self, item): - return item in self._store - - def __len__(self): - return len(self._store) - - def add(self, workload: str, device: str, scheduler: str, metrics: dict): - """Adds a new key to the datastore. - - :param workload: The workload. - :param device: The device. - :param scheduler: The scheduler. - :param metrics: The metrics. - """ - key = (workload, device, scheduler) - if key not in self._store: - self._store[key] = {'workload': workload, 'device': device, 'scheduler': scheduler, 'key': key, - 'metrics': metrics} - - def get(self, workload: str, device: str, scheduler: str): - """Retrieves a single item matching the given key. - - :param workload: The workload. - :param device: The device. - :param scheduler: The scheduler. - :return: The retrieved item. - :exception KeyError: Raised if key not found. - """ - key = (workload, device, scheduler) - if key not in self._store: - raise KeyError("Unable to find key: (%s, %s, %s)" % (workload, device, scheduler)) - - return self._store[key] - - def get_all(self, **kwargs): - """Retrieves all items with keys matching the given optional kwargs (workload, device, scheduler). - - :param kwargs: The following optional kwargs can be specified for lookups (workload, device, scheduler). Only - the specified key parts will be matched on. If none are specified, all items are retrieved. - :return: A list of matched items. - """ - workload = None - if 'workload' in kwargs: - workload = kwargs['workload'] - - device = None - if 'device' in kwargs: - device = kwargs['device'] - - scheduler = None - if 'scheduler' in kwargs: - scheduler = kwargs['scheduler'] - - items = [] - - for key, value in self._store.items(): - if workload and key[0] != workload: - continue - - if device and key[1] != device: - continue - - if scheduler and key[2] != scheduler: - continue - - items.append(value) - - return items - - -class Metrics: - """A group of metrics for a particular workload.""" - - __output_initialized = False - - def __init__(self, workload: str): - self.workload = workload - self._metrics = [] - - def add_metrics(self, metrics: dict): - """Adds new metrics. - - :param metrics: The metrics. Expects mapping of metric name to metric value (int or float).""" - self._metrics.append(metrics) - - def average_metrics(self) -> dict: - """Averages the metrics into a new dictionary. - - :return: The averaged metrics. - """ - averaged_metrics = dict() # The averaged metrics - metric_frequency = dict() # The frequency of each metric - - # Sums the metrics then divides each by their frequency - for metric in self._metrics: - for key, value in metric.items(): - metric_frequency.setdefault(key, 0) - metric_frequency[key] += 1 - - averaged_metrics.setdefault(key, 0) - averaged_metrics[key] += value - - for key in averaged_metrics: - averaged_metrics[key] = averaged_metrics[key] / metric_frequency[key] - - return averaged_metrics - - @staticmethod - def average_metric(metrics: dict, names: tuple): - """Returns the average of the metrics. - - :param metrics: The metrics dictionary. - :param names: The name of the metrics to average. - :return: The average of the metrics. - """ - count = 0 - summation = 0 - for name in names: - if name in metrics and metrics[name] > 0: - summation += metrics[name] - count += 1 - - if count == 0: - return 0 - else: - return summation / count - - @staticmethod - def gather_workload_metrics(workload_out: str, workload: str) -> dict: - """Parses workload outputs and returns relevant metrics. - - :param workload_out: The workload output. - :param workload: The workload. - :return: A dictionary of metrics and their values. - """ - ret = defaultdict(int) - - if workload == 'fio': - data = json.loads(workload_out, encoding='utf-8') - - bwrc, bwwc = 0, 0 - crc, cwc = 0, 0 - src, swc = 0, 0 - lrc, lwc = 0, 0 - iopsr, iopsw = 0, 0 - iokb = 0, - - for job in data['jobs']: - ret['throughput-read'] += float(job['read']['bw']) / 1024 - if job['read']['bw'] > 0: - bwrc += 1 - log('Grabbing metric %s: %s' % ('throughput-read', job['read']['bw'] / 1024)) - - ret['throughput-write'] += float(job['write']['bw']) / 1024 - if job['write']['bw'] > 0: - bwwc += 1 - log('Grabbing metric %s: %s' % ('throughput-write', job['write']['bw'] / 1024)) - - ret['clat-read'] += float(job['read']['clat_ns']['mean']) - if job['read']['clat_ns']['mean'] > 0: - crc += 1 - log('Grabbing metric %s: %s' % ('clat-read', job['read']['clat_ns']['mean'])) - - ret['clat-write'] += float(job['write']['clat_ns']['mean']) - if job['write']['clat_ns']['mean'] > 0: - cwc += 1 - log('Grabbing metric %s: %s' % ('clat-write', job['write']['clat_ns']['mean'])) - - ret['slat-read'] += float(job['read']['slat_ns']['mean']) - if job['read']['slat_ns']['mean'] > 0: - src += 1 - log('Grabbing metric %s: %s' % ('slat-read', job['read']['slat_ns']['mean'])) - - ret['slat-write'] += float(job['write']['slat_ns']['mean']) - if job['write']['slat_ns']['mean'] > 0: - swc += 1 - log('Grabbing metric %s: %s' % ('slat-write', job['write']['slat_ns']['mean'])) - - ret['lat-read'] += float(job['read']['lat_ns']['mean']) - if job['read']['lat_ns']['mean'] > 0: - lrc += 1 - log('Grabbing metric %s: %s' % ('lat-read', job['read']['lat_ns']['mean'])) - - ret['lat-write'] += float(job['write']['lat_ns']['mean']) - if job['write']['lat_ns']['mean'] > 0: - lwc += 1 - log('Grabbing metric %s: %s' % ('lat-write', job['write']['lat_ns']['mean'])) - - ret['iops-read'] += float(job['read']['iops']) - if job['read']['iops'] > 0: - iopsr += 1 - log('Grabbing metric %s: %s' % ('iops-read', job['read']['iops'])) - - ret['iops-write'] += float(job['write']['iops']) - if job['write']['iops'] > 0: - iopsw += 1 - log('Grabbing metric %s: %s' % ('iops-write', job['write']['iops'])) - - ret['io-kbytes'] += float(job['read']['io_kbytes']) - if job['read']['io_kbytes'] > 0: - log('Grabbing metric %s: %s' % ('io-kbytes', job['read']['io_kbytes'])) - - ret['io-kbytes'] += float(job['write']['io_kbytes']) - if job['write']['io_kbytes'] > 0: - log('Grabbing metric %s: %s' % ('io-kbytes', job['write']['io_kbytes'])) - - ret['io-depth'] = int(job['job options']['iodepth']) - - # Compute averages - if bwrc > 0: ret['throughput-read'] /= bwrc - if bwwc > 0: ret['throughput-write'] /= bwwc - if crc > 0: ret['clat-read'] /= crc - if cwc > 0: ret['clat-write'] /= cwc - if src > 0: ret['slat-read'] /= src - if swc > 0: ret['slat-write'] /= swc - if lrc >0: ret['lat-read'] /= lrc - if lwc > 0: ret['lat-write'] /= lwc - if iopsr > 0: ret['iops-read'] /= iopsr - if iopsw > 0: ret['iops-write'] /= iopsw - - # Adjust values to be in µs - ret['clat-read'] /= 10**3 - ret['clat-write'] /= 10**3 - ret['slat-read'] /= 10**3 - ret['slat-write'] /= 10**3 - ret['lat-read'] /= 10 ** 3 - ret['lat-write'] /= 10 ** 3 - else: - print_detailed('Unable to interpret workload %s' % workload) - - return ret - - @staticmethod - def gather_metrics(blkparse_out: str, btt_out: str, workload_out: str, workload: str) -> dict: - """Parses command outputs and returns relevant metrics. - - :param blkparse_out: The blkparse command output. - :param btt_out: The btt command output. - :param workload_out: The workload output. - :param workload: The workload. - :return: A dictionary of metrics and their values. - """ - metrics = dict() - - # blkparse - # throughput_read = Mem.re_blkparse_throughput_read.findall(blkparse_out) - - # if throughput_read: - # metrics['throughput-read'] = float(throughput_read[0]) / 1024 - # log('Grabbing metric %s: %s' % ('throughput-read', metrics['throughput-read'])) - - # throughput_write = Mem.re_blkparse_throughput_write.findall(blkparse_out) - - # if throughput_write: - # metrics['throughput-write'] = float(throughput_write[0]) / 1024 - # log('Grabbing metric %s: %s' % ('throughput-write', metrics['throughput-write'])) - - # btt - d2c = Mem.re_btt_d2c.findall(btt_out) - - if d2c: - metrics['d2c'] = float(d2c[0]) * 10**6 # µs - log('Grabbing metric %s: %s' % ('d2c', metrics['d2c'])) - - q2c = Mem.re_btt_q2c.findall(btt_out) - - if q2c: - metrics['q2c'] = float(q2c[0]) * 10**6 # µs - log('Grabbing metric %s: %s' % ('q2c', metrics['q2c'])) - - workload_metrics = Metrics.gather_workload_metrics(workload_out, workload) - - metrics = defaultdict(int, {**metrics, **workload_metrics}) - - return metrics - - @staticmethod - @ignore_exception() - @log_around(exception_message='Unable to output metrics to console!') - def print(job_name: str, workload: str, scheduler: str, device: str, metrics: dict): - """Prints metric information to STDOUT. - - :param job_name: The name of the job. - :param workload: The workload. - :param scheduler: The scheduler. - :param device: The device. - :param metrics: The metrics. - """ - print_and_log('%s [%s]:' % (job_name, workload)) - print_and_log(' (%s) (%s):' % (scheduler, device)) - print_and_log(' Latency [µs]: (read): %.2f (write): %.2f' % (metrics['lat-read'], metrics['lat-write'])) - print_and_log(' Submission Latency [µs]: (read): %.2f (write): %.2f' % (metrics['slat-read'], metrics['slat-write'])) - print_and_log(' Completion Latency [µs]: (read): %.2f (write): %.2f' % (metrics['clat-read'], metrics['clat-write'])) - print_and_log(' File System Latency [µs]: (read): %.2f (write): %.2f' % (metrics['fslat-read'], metrics['fslat-write'])) - print_and_log(' Block Layer Latency [µs]: %.2f' % metrics['bslat']) - print_and_log(' Device Latency [µs]: %.2f' % metrics['d2c']) - print_and_log(' IOPS: (read) %.2f (write) %.2f' % (metrics['iops-read'], metrics['iops-write'])) - print_and_log(' Throughput [1024 MB/s]: (read) %.2f (write) %.2f' % (metrics['throughput-read'], metrics['throughput-write'])) - print_and_log(' Total IO [KB]: %.2f' % metrics['io-kbytes']) - - @staticmethod - @ignore_exception() - @log_around(exception_message='Unable to output metrics to file!') - def output(metrics: dict): - """Prints metric information in csv format to output file. - - :param metrics: The metrics. - """ - if Mem.output_file is not None: - if not Metrics.__output_initialized: - Metrics.__init_output() - - with open(Mem.output_file, 'a') as file: - first = True - for column in Mem.output_column_order: - if first: - first = False - else: - file.write(',') - - val = metrics[column] - - if type(val) is float: - file.write('%0.2f' % val) - else: - file.write(str(val)) - file.write('\n') - - @staticmethod - @ignore_exception() - @log_around(exception_message='Unable to create output file!') - def __init_output(): - """Initializes output by creating file if doesn't already exist and adding header.""" - Metrics.__output_initialized = True - - # Create file if doesn't exist - if not os.path.isfile(Mem.output_file): - # Add header - with open(Mem.output_file, 'w') as file: - file.write(','.join(Mem.output_column_order)) - file.write('\n') - -# endregion - - -# region commands -@log_around(after_message='Changed scheduler successfully', - exception_message='Unable to change scheduler', - ret_validity=True) -def change_scheduler(scheduler: str, device: str): - """Changes the I/O scheduler for the given device. - - :param scheduler: The I/O scheduler. - :param device: The device. - :return: Returns True if successful, else False. - """ - log('Changing scheduler for device %s to %s' % (device, scheduler)) - - command = 'bash -c "echo %s > /sys/block/%s/queue/scheduler"' % (scheduler, Mem.re_device.findall(device)[0]) - - out, rc = run_command(command) - - return rc == 0 - - -@log_around('Validating required tracing dependencies are installed', - 'Verified required tracing dependencies are required', - 'Missing required tracing dependencies', - True) -def check_trace_commands() -> bool: - """Validates whether the required tracing commands exists on the system. - - :return: Returns True if commands exists, else False. - """ - if not command_exists('blktrace'): - print_detailed('blktrace is not installed. Please install via \'sudo apt install blktrace\'') - return False - - if not command_exists('blkparse'): # Included with blktrace - print_detailed('blkparse is not installed. Please install via \'sudo apt install blktrace\'') - return False - - if not command_exists('btt'): # Included with blktrace - print_detailed('btt is not installed. Please install via \'sudo apt install blktrace\'') - return False - - return True - - -@log_around(exception_message='Unable to clean up files') -def cleanup_files(*files): - """Removes the specified file, or files if multiple are given. - - :param files: Files to remove.. - """ - if not Mem.cleanup: # Only cleanup if specified - return - - log('Cleaning up files') - - for file in files: - log('Removing files %s' % file) - run_system_command('rm -f %s' % file) - - -@log_around(before_message='Clearing caches', exception_message='Unable to clear caches') -def clear_caches(device: str): - """Clears various data caches. Should be run before each benchmark. - - :param device: The device to clear the caches for. - """ - # Writes any data buffered in memory out to disk - run_system_command('sync') - - # Drops clean caches - run_system_command('echo 3 > /proc/sys/vm/drop_caches') - - # Calls block device ioctls to flush buffers - run_system_command('blockdev --flushbufs %s' % device) - - # Flushes the on-drive write cache buffer - run_system_command('hdparm -F %s' % device) - - -@log_around(after_message='Verified dependency exists', - exception_message='Missing dependency', - ret_validity=True) -def command_exists(command: str) -> bool: - """Returns whether the given command exists on the system. - - :param command: The command. - :return: Returns True if exists, else False. - """ - log('Checking if dependency %s exists' % command) - - rc = run_system_command('command -v %s' % command) - - return rc == 0 - - -@log_around(exception_message='Unable to retrieve major,minor information', ret_validity=True) -def get_device_major_minor(device: str) -> str: - """Returns a string of the major, minor of a given device. - - :param device: The device. - :return: A string of major,minor. - """ - log('Retrieving major,minor for device %s' % device) - - out, _ = run_command('stat -c \'%%t,%%T\' %s' % device) - - return out if not out else out.strip() - - -def get_schedulers(device: str) -> list: - """Returns a list of available schedulers for a given device. - - :param device: The device. - :return: Returns a list of schedulers. - """ - log('Retrieving schedulers for device %s' % device) - - matches = Mem.re_device.findall(device) - - if not matches: - log('Unable to find schedulers for device') - return [] - - out, rc = run_command('cat /sys/block/%s/queue/scheduler' % matches[0]) - - if rc != 0: - log('Unable to find schedulers for device') - return [] - - ret = out.replace('[', '').replace(']', '') - - log('Found the following schedulers for device %s: %s' % (device, ret)) - - return ret.split() - - -@log_around(before_message='Validating proposed schedulers', - exception_message='Unable to validate proposed schedulers') -def get_valid_schedulers(device: str, proposed_schedulers: list) -> list: - """Returns a list of schedulers that are valid for a given device and set of proposed schedulers. - - :param device: The device. - :param proposed_schedulers: The proposed schedulers. - :return: Returns a list of schedulers. - """ - valid_schedulers = [] - - available_schedulers = set(get_schedulers(device)) - - for scheduler in proposed_schedulers: - if scheduler in available_schedulers: - valid_schedulers.append(scheduler) - - return valid_schedulers - - -@ignore_exception(FileNotFoundError, False) -@ignore_exception(TypeError, False) -@log_around(after_message='Device is a valid block device', - exception_message='Device is not a valid block device') -def is_block_device(device: str) -> bool: - """Returns whether the given device is a valid block device. - - :param device: The device. - :return: Returns True if is a valid block device, else False. - """ - log('Checking if device %s is a valid block device' % device) - - info = os.stat(device) - return stat.S_ISBLK(info.st_mode) - - -@log_around(after_message='Device is a rotational device', - exception_message='Device is not a rotational device', - ret_validity=True) -def is_rotational_device(device: str) -> bool: - """Returns whether the given device is a rotational device. - - :param device: The device. - :return: Returns True if is a rotational device, else False. - """ - log('Checking whether device %s is a rotational device' % device) - - matches = Mem.re_device.findall(device) - - if not matches: - return False - - out, rc = run_command('cat /sys/block/%s/queue/rotational' % matches[0]) - - if rc != 0: - return False - - return int(out) == 1 - - -@log_around(after_message='Setting is valid', - exception_message='Setting is invalid', - ret_validity=True) -def is_valid_setting(setting: str, header: str) -> bool: - """Returns whether the config setting is valid. - - :return: Returns True if setting is valid, else False. - """ - log('Checking whether setting %s under %s is valid' % (setting, header)) - - if not header: - return False - - if not setting: - return False - - if header == Mem.GLOBAL_HEADER: - return setting in Mem.valid_global_settings - else: - return setting in Mem.valid_job_settings - - -@log_around(after_message='Workload is valid', - exception_message='Workload is invalid', - ret_validity=True) -def is_valid_workload(workload: str) -> bool: - """Returns whether the given workload is valid. - - :param workload: The workload. - :return: Returns True if valid, else False. - """ - log('Checking whether workload %s is valid' % workload) - - if workload not in Mem.valid_workloads: - return False - - if not command_exists(workload): - return False - - return True - - -def run_command(command: str, inp: str='', ignore_output: bool = False) -> (str, int): - """Runs a command via subprocess communication. - - :param command: The command. - :param inp: (OPTIONAL) Command input. - :param ignore_output: (OPTIONAL) Whether to ignore the output. Defaults to False. - :return: A tuple containing (the output, the return code). - """ - log('Running command %s with input %s' % (command, inp)) - - try: - args = shlex.split(command) - - p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, - preexec_fn=os.setsid) - - Mem.current_processes.add((command, p)) - - if ignore_output: - rc = p.wait() - - Mem.current_processes.clear() - - if rc != 0: - print_detailed('Error, return code is not zero!') - - return '', rc - else: - out, err = p.communicate(inp) - - rc = p.returncode - - Mem.current_processes.clear() - - if err: - print_detailed(err.decode('utf-8')) - - return out.decode('utf-8'), rc - except (ValueError, subprocess.CalledProcessError, FileNotFoundError) as err: - print_detailed(err) - return None, None - finally: - Mem.current_processes.clear() - - -def run_parallel_commands(command_map: list, max_concurrent: int=multiprocessing.cpu_count(), - abort_on_failure: bool=True): - """Runs multiple commands in parallel via subprocess communication. Commands are run in order of delay, with their - respective delays considered (useful when commands like fio take time to generate a file before running). - - A single failed process results in the remaining being stopped. - - :param command_map: A command mapping which contains a list of tuples containing (command name, command delay, - the command itself). - :param max_concurrent: The maximum number of concurrent processes. - :param abort_on_failure: Whether to abort if a single process failures, otherwise continues. Defaults to True. - :return: A dictionary where key = command name and value = tuple of (the output, the return code). - """ - log('Running commands in parallel') - - if max_concurrent < 1: - print_detailed('Maximum concurrent processes must be > 0') - return None - - Mem.current_processes.clear() - - completed_processes = set() - - last_delay = 0 - - for command_name, delay, command in sorted(command_map, key=lambda x: x[1]): - try: - # Delay command execution based on specified delay - # Note: This isn't quite exact, due to timing issues and the concurrency limit - if delay > last_delay: - time.sleep(delay - last_delay) - last_delay = delay - - log('Running command %s' % command) - - args = shlex.split(command) - - p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, - preexec_fn=os.setsid) - - Mem.current_processes.add((command_name, p)) - except (ValueError, subprocess.CalledProcessError, FileNotFoundError) as err: - print_detailed(err) - if abort_on_failure: - break - - # Limit the number of threads - while len(Mem.current_processes) >= max_concurrent: - time.sleep(0.5) - - finished_processes = get_finished_processes(Mem.current_processes) - - Mem.current_processes.difference_update(finished_processes) - completed_processes.update(finished_processes) - - if abort_on_failure: - failed_processes = get_failed_processes(finished_processes) - - if failed_processes: # Something failed, abort! - print_processes(failed_processes) - kill_processes(Mem.current_processes) - break - else: - # Wait for processes to finish - while len(Mem.current_processes) > 0: - time.sleep(0.5) - - finished_processes = get_finished_processes(Mem.current_processes) - - Mem.current_processes.difference_update(finished_processes) - completed_processes.update(finished_processes) - - if abort_on_failure: - failed_processes = get_failed_processes(finished_processes) - - if failed_processes: # Something failed, abort! - print_processes(failed_processes) - kill_processes(Mem.current_processes) - return None - - ret = dict() - - # Grab outputs from completed processes - for command_name, process in completed_processes: - out, err = process.communicate() - - rc = process.returncode - - if err: - print_detailed(err.decode('utf-8')) - - ret[command_name] = (out.decode('utf-8'), rc) - - return ret - - # We got here because we aborted, continue the abortion... - failed_processes = get_failed_processes(Mem.current_processes) - print_processes(failed_processes) - - kill_processes(Mem.current_processes) - - return None - - -@log_around(exception_message='Error occurred running command') -def run_system_command(command: str, silence: bool=True) -> int: - """Runs a system command. - - :param command: The command. - :param silence: (OPTIONAL) Whether to silence the console output. Defaults to True. - :return: The return code. - """ - if silence: - command = '%s >/dev/null 2>&1' % command - - log('Running command %s' % command) - - rc = os.system(command) - return rc - - -@log_around('Validating jobs', 'Valid jobs found', 'All jobs are invalid', True) -def validate_jobs() -> bool: - """Returns whether each job is valid. - - :return: Returns True if all are valid, else False. - """ - job_index = 0 - while job_index < len(Mem.jobs): - job = Mem.jobs[job_index] - - log('Validating job %s' % job.name) - - # Fill in missing settings from globals - job.fill_missing(Mem) - - # Ensure job has required properties - if not job.is_valid(): - ip = ', '.join(job.get_invalid_props()) - print_detailed('Job %s is missing the required settings: %s' % (job.name, ip)) - if Mem.continue_on_failure: - Mem.jobs.pop(job_index) - continue - else: - return False - - if not is_valid_workload(job.workload): - print_detailed('%s is not installed. Please install the tool before use.' % job.workload) - if Mem.continue_on_failure: - Mem.jobs.pop(job_index) - continue - else: - return False - - if not is_block_device(job.device): - print_detailed('The device %s is not a valid block device.' % job.device) - if Mem.continue_on_failure: - Mem.jobs.pop(job_index) - continue - else: - return False - - # We'll allow schedulers to be defined that don't exist for every device - # So no checks here... - - job_index += 1 - - return len(Mem.jobs) > 0 # At least 1 job required -# endregion - - -# region command-line -def usage(): - """Displays command-line information.""" - name = os.path.basename(__file__) - print('%s %s' % (name, __version__)) - print('Usage: %s [-c] [-l] [-o ] [-r ] [-v] [-x]' % name) - print('Command Line Arguments:') - print(' : The configuration file to use.') - print('-c : (OPTIONAL) The application will continue in the case of a job failure.') - print('-l : (OPTIONAL) Logs debugging information to an iobs.log file.') - print('-o : (OPTIONAL) Outputs metric information to a file.') - print('-r : (OPTIONAL) Used to retry a job more than once if failure occurs. Defaults to 1.') - print('-v : (OPTIONAL) Prints verbose information to the STDOUT.') - print('-x : (OPTIONAL) Attempts to clean up intermediate files.') - - -@log_around(before_message='Parsing command-line arguments', exception_message='Unable to parse arguments', - ret_validity=True) -def parse_args(argv: list) -> bool: - """Parses the supplied arguments and persists in memory. - - :param argv: A list of arguments. - :return: Returns a boolean as True if parsed correctly, otherwise False. - """ - try: - opts, args = getopt(argv, 'ghlo:r:vx') - - for opt, arg in opts: - if opt == '-c': - Mem.continue_on_failure = True - elif opt == '-h': - return False - elif opt == '-l': - Mem.log = True - elif opt == '-o': - Mem.output_file = arg - elif opt == '-r': - conv_value = ignore_exception(ValueError, -1)(int)(arg) - - if conv_value < 1: - print_detailed('Retry count must be >= 1, given %s' % arg) - return False - - Mem.retry = conv_value - elif opt == '-v': - Mem.verbose = True - elif opt == '-x': - Mem.cleanup = True - return True - except GetoptError as err: - print_detailed(err) - return False - - -def parse_config_file(file_path: str) -> bool: - """Parses the supplied file and persists data into memory. - - :param file_path: The file. - :return: Returns True if settings are valid, else False. - """ - log('Parsing configuration file: %s' % file_path) - Mem.config_file = file_path - - if not os.path.isfile(Mem.config_file): - sys.exit('File not found: %s' % Mem.config_file) - - config = configparser.ConfigParser() - - - try: - config.read(file_path, 'utf-8') - except configparser.ParsingError as err: - print_detailed('Invalid syntax in config file found!') - log(err) - return False - - for section in config.sections(): - if section == Mem.GLOBAL_HEADER: - for key, value in config[section].items(): - if not is_valid_setting(key, section): - print_detailed('Invalid syntax in config file found: %s=%s' % (key, value)) - return False - - try: - setattr(Mem, key, value) - except ValueError: - print_detailed('Invalid syntax in config file found: %s=%s' % (key, value)) - return False - else: - Mem.jobs.append(Job(section)) - for key, value in config[section].items(): - if not is_valid_setting(key, section): - print_detailed('Invalid syntax in config file found: %s=%s' % (key, value)) - return False - - try: - setattr(Mem.jobs[-1], key, value) - except ValueError: - print_detailed('Invalid syntax in config file found: %s=%s' % (key, value)) - return False - - return True -# endregion - - -# region processes -def get_failed_processes(processes: set) -> set: - """Returns the processes which are failed. - - :param processes: A set of tuples of command names and processes. - :return: A set of failed processes. - """ - failed_processes = set() - - for command_name, process in processes: - rc = process.poll() - - if rc is not None: # Done processing - if rc != 0: # Return code other than 0 indicates error - failed_processes.add((command_name, process)) - - return failed_processes - - -def get_finished_processes(processes: set) -> set: - """Returns the processes which are finished. - - :param processes: A set of tuples of command names and processes. - :return: A set of finished processes. - """ - finished_processes = set() - - for command_name, process in processes: - rc = process.poll() - - if rc is not None: # Done processing - finished_processes.add((command_name, process)) - - return finished_processes - - -@log_around('Killing processes', - 'Killed all processes', - 'Unable to kill all processes') -def kill_processes(processes: set): - """Kills the processes. - - :param processes: A set of tuples of command names and processes. - """ - for command_name, process in processes: - try: - log('Killing process %s' % process) - os.killpg(os.getpgid(process.pid), signal.SIGTERM) - except: - pass - - -def print_processes(processes: set): - """Prints the each processes's output. - - :param processes: A set of tuples of command names and processes. - """ - for command_name, process in processes: - out, err = process.communicate() - if out: - print_detailed(out.decode('utf-8')) - if err: - print_detailed(err.decode('utf-8')) -# endregion - - -@log_around('Beginning program execution', 'Finishing program execution', 'Program encountered critical error') -def main(argv: list): - # Help flag dominates all args - if '-h' in argv: - usage() - sys.exit(1) - - # Validate privileges - if os.getuid() != 0: - print('Script must be run with administrative privileges. Try sudo %s' % __file__) - sys.exit(1) - - # Set logging as early as possible - if '-l' in argv: - logging.basicConfig(filename='iobs.log', level=logging.DEBUG, format='%(asctime)s - %(message)s') - Mem.log = True - - if '-v' in argv: - Mem.verbose = True - - # Validate os - ps = platform.system() - if ps != 'Linux': - print_detailed('OS is %s, must be Linux' % ps) - sys.exit(1) - - if len(argv) == 0: - usage() - sys.exit(1) - - # Validate settings - if not parse_config_file(argv[0]): - sys.exit(1) - - if not validate_jobs(): - sys.exit(1) - - # Validate arguments - if not parse_args(argv[1:]): - usage() - sys.exit(1) - - if not check_trace_commands(): - sys.exit(1) - - # Beginning running jobs - if not Mem.process_jobs(): - sys.exit(1) - - -if __name__ == '__main__': - # Add signal handlers for graceful termination - signal.signal(signal.SIGTERM, sig_handler) - signal.signal(signal.SIGINT, sig_handler) - - main(sys.argv[1:]) diff --git a/iobs/commands/execute.py b/iobs/commands/execute.py new file mode 100644 index 0000000..aba47e0 --- /dev/null +++ b/iobs/commands/execute.py @@ -0,0 +1,189 @@ +#!/usr/bin/python3 +# Copyright (c) 2018, UofL Computer Systems Lab. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without event the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import argparse +import logging +import os +import platform + +from iobs.config import parse_config_file +from iobs.errors import ( + InvalidOSError, + InvalidPrivilegesError, + IOBSBaseException +) +from iobs.settings import SettingsManager +from iobs.output import printf, PrintType + + +def check_args(args): + """Checks the command-line arguments and sets settings. + + Args: + args: The arguments to check. + """ + SettingsManager.set('output_directory', args.output_directory) + os.makedirs(args.output_directory, exist_ok=True) + + if args.log_file: + SettingsManager.set('log_enabled', True) + log_level = get_log_level(args.log_level) + log_path = os.path.join(os.getcwd(), args.log_file) + logging.basicConfig(filename=log_path, level=log_level, + format='%(asctime)s - %(message)s') + else: + SettingsManager.set('log_enabled', False) + + SettingsManager.set('silent', args.silent) + SettingsManager.set('retry_count', args.retry_count) + SettingsManager.set('continue_on_failure', args.continue_on_failure) + + +def get_log_level(log_level): + """Converts the `log_level` into logging level. + + Args: + log_level: The level to convert. + + Returns: + A logging level. + """ + if log_level == 1: + return logging.DEBUG + if log_level == 2: + return logging.INFO + return logging.ERROR + + +def validate_os(): + """Checks whether the required operating system is in use. + + Raises: + InvalidOSError: If OS is not Linux. + """ + ps = platform.system() + if ps != 'Linux': + raise InvalidOSError('OS is {}, must be Linux.'.format(ps)) + + +def validate_privileges(): + """Checks whether the script is ran with administrative privileges. + + Raises: + InvalidPrivilegesError: If script isn't ran with sudo privileges. + """ + if os.getuid() != 0: + raise InvalidPrivilegesError( + 'Script must be run with administrative privileges.' + ) + + +def execute(args): + """Executes workloads. + + Args: + args: The parsed command-line arguments. + + Returns: + 0 if successful. + + Raises: + IOBSBaseException: If error occurs and `continue_on_failure` not set. + """ + printf('Beginning program execution...', + print_type=PrintType.NORMAL | PrintType.INFO_LOG) + + check_args(args) + validate_os() + validate_privileges() + + for i, input_file in enumerate(args.inputs): + try: + printf('Processing input file {} ({} of {})' + .format(input_file, i + 1, len(args.inputs))) + + configuration = parse_config_file(input_file) + configuration.validate() + configuration.process() + except IOBSBaseException as err: + if not SettingsManager.get('continue_on_failure'): + raise err + + printf('input file {} failed all retries. Continuing execution ' + 'of remaining files...\n{}'.format(input_file, err), + print_type=PrintType.ERROR | PrintType.ERROR_LOG) + + printf('Finishing program execution...', + print_type=PrintType.NORMAL | PrintType.INFO_LOG) + + return 0 + + +def main(args): + parser = argparse.ArgumentParser(prog='iobs execute') + parser.add_argument( + 'inputs', + nargs='+', + metavar='input', + help='The configuration files to execute.' + ) + parser.add_argument( + '-o', '--output-directory', + dest='output_directory', + default=os.getcwd(), + help='The output directory for output and log files. Defaults to the ' + 'current working directory.' + ) + parser.add_argument( + '-l', '--log-file', + dest='log_file', + help='The file to log information to.' + ) + parser.add_argument( + '--log-level', + dest='log_level', + choices=[1, 2, 3], + default=1, + type=int, + help='The level of information to which to log: 1 (Debug), ' + '2 (Info), 3(Error). Defaults to 2.' + ) + parser.add_argument( + '-s', '--silent', + default=False, + action='store_true', + help='Silences output to STDOUT.' + ) + parser.add_argument( + '-r', '--retry-count', + dest='retry_count', + default=1, + type=int, + help='Number of times to retry a failed workload. Defaults to 1.' + ) + parser.add_argument( + '-c', '--continue-on-failure', + dest='continue_on_failure', + default=False, + action='store_true', + help='If a input fails, continues executing other inputs; otherwise ' + 'exits the program.' + ) + + args = parser.parse_args(args) + return execute(args) diff --git a/iobs/config.py b/iobs/config.py new file mode 100644 index 0000000..ddc9ec4 --- /dev/null +++ b/iobs/config.py @@ -0,0 +1,1115 @@ +# Copyright (c) 2018, UofL Computer Systems Lab. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without event the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +from abc import ABC, abstractmethod +from configparser import ConfigParser, ParsingError +import itertools +import json +import os + +from iobs.errors import ( + ConfigNotFoundError, + InvalidConfigError, + InvalidSettingError, + JobExecutionError, + OutputFileError, + OutputFormatError, + OutputParsingError, + RetryCountExceededError, + UndefinedWorkloadTypeError +) +from iobs.output import printf, PrintType +from iobs.process import ( + change_scheduler, + clear_caches, + is_block_devices, + run_command, + validate_schedulers +) +from iobs.settings import ( + is_valid_workload_type, + get_constant, + get_formatter, + match_regex, + SettingAttribute, + SettingsManager, + validate_settings +) +from iobs.util import cast_bool, try_split + + +# region Config File +def parse_config_file(input_file): + """Parses the supplied configuration file. + + Args: + input_file: The file. + + Returns: + A Configuration. + + Raises: + InvalidConfigError: If unable to parse configuration. + """ + printf('Parsing configuration file {}'.format(input_file), + print_type=PrintType.DEBUG_LOG) + + if not os.path.isfile(input_file): + raise ConfigNotFoundError('Config file {} not found'.format(input_file)) + + config_parser = ConfigParser() + + try: + config_parser.read(input_file, 'utf-8') + except ParsingError as err: + raise InvalidConfigError( + 'Invalid syntax in config file {}\n{}'.format(input_file, err) + ) + + global_header = get_constant('config_header_global') + output_header = get_constant('config_header_output') + template_header = get_constant('config_header_template') + + workload_type = get_workload_type(config_parser) + workload_configuration_type = get_workload_configuration_type(workload_type) + global_configuration_type = get_global_configuration_type(workload_type) + output_configuration_type = get_output_configuration_type(workload_type) + + global_configuration = global_configuration_type() + output_configuration = output_configuration_type(input_file) + template_configuration = TemplateConfiguration() + configuration = Configuration( + input_file, + workload_type, + global_configuration, + output_configuration, + template_configuration + ) + + for section in config_parser.sections(): + if section == global_header: + parse_section(config_parser, section, global_configuration) + elif section == output_header: + parse_section(config_parser, section, output_configuration) + elif section == template_header: + parse_section(config_parser, section, template_configuration) + else: + workload_configuration = workload_configuration_type(section) + parse_section(config_parser, section, workload_configuration) + configuration.add_workload_configuration(workload_configuration) + + printf('Configuration file {} parsed successfully'.format(input_file), + print_type=PrintType.DEBUG_LOG) + + return configuration + + +def parse_section(config_parser, section, config): + """Parses a section of a config file. + + Args: + config_parser: The config parser. + section: The section name. + config: The Configuration object. + """ + printf('Parsing section {}'.format(section), + print_type=PrintType.DEBUG_LOG) + + for key, value in config_parser[section].items(): + config.add_setting(key, value) + + +def get_workload_type(config_parser): + """Retrieves the workload type from the config file. + + Args: + config_parser: The config parser. + + Returns: + The workload type. + + Raises: + InvalidConfigError: If workload type not in config or valid. + """ + global_header = get_constant('config_header_global') + if not config_parser.has_section(global_header): + raise InvalidConfigError( + 'Missing required {} section in config'.format(global_header) + ) + + global_section = config_parser[global_header] + if 'workload_type' not in global_section: + raise InvalidConfigError('workload_type not defined in config') + + workload_type = global_section['workload_type'] + valid_workloads = get_constant('valid_workload_types') + + if workload_type not in valid_workloads: + raise InvalidConfigError( + 'workload_type {} is not valid'.format(workload_type) + ) + + return workload_type +# endregion + + +# region Factories +def get_global_configuration_type(workload_type): + if workload_type == 'fio': + return FIOGlobalConfiguration + + raise UndefinedWorkloadTypeError( + 'workload_type {} is not defined'.format(workload_type) + ) + + +def get_job_type(workload_type): + if workload_type == 'fio': + return FIOJob + + raise UndefinedWorkloadTypeError( + 'workload_type {} is not defined'.format(workload_type) + ) + + +def get_output_configuration_type(workload_type): + if workload_type == 'fio': + return FIOOutputConfiguration + + raise UndefinedWorkloadTypeError( + 'workload_type {} is not defined'.format(workload_type) + ) + + +def get_workload_configuration_type(workload_type): + if workload_type == 'fio': + return FIOWorkloadConfiguration + + raise UndefinedWorkloadTypeError( + 'workload_type {} is not defined'.format(workload_type) + ) +# endregion + + +# region Configuration +class Configuration: + """A Configuration representing a single config input. + + Args: + input_file: The input file. + workload_type: The workload type. + global_configuration: The GlobalConfiguration. + output_configuration: The OutputConfiguration. + template_configuration: The TemplateConfiguration. + """ + def __init__(self, input_file, workload_type, global_configuration, + output_configuration, template_configuration): + self._workload_type = workload_type + self._input_file = input_file + self._global_configuration = global_configuration + self._output_configuration = output_configuration + self._template_configuration = template_configuration + self._workload_configurations = [] + + def add_workload_configuration(self, workload_configuration): + """Adds a WorkloadConfiguration to process. + + Args: + workload_configuration: The workload configuration. + """ + self._workload_configurations.append(workload_configuration) + + def process(self): + """Processes the configuration.""" + printf('Processing input file {}'.format(self._input_file), + print_type=PrintType.DEBUG_LOG) + + for wc in self._workload_configurations: + wc.process(self._output_configuration, self._template_configuration, + self._global_configuration) + + def validate(self): + """Validates the configuration.""" + printf('Validating input file {}'.format(self._input_file), + print_type=PrintType.DEBUG_LOG) + + self._global_configuration.validate() + self._output_configuration.validate() + self._template_configuration.validate() + + for wc in self._workload_configurations: + wc.validate() + + +class ConfigSectionBase(ABC): + """Base class for configurations.""" + def __init__(self): + self._settings = self._get_settings() + + @abstractmethod + def add_setting(self, setting, value): + """Adds a setting to the configuration object. + + Args: + setting: The setting. + value: The value. + """ + + @abstractmethod + def _get_settings(self): + """Retrieves the SettingAttributes for the configuration object. + + Returns: + A dictionary mapping of setting names to SettingAttributes. + """ + + def validate(self): + """Validates the settings.""" + validate_settings(self, self._settings) + + +# region Global Configuration +class GlobalConfiguration(ConfigSectionBase): + """Global Configuration for `global` section of config.""" + def add_setting(self, setting, value): + """Adds a setting to the configuration object. + + Args: + setting: The setting. + value: The value. + + Raises: + InvalidSettingError: If setting is not defined in `_get_settings`. + """ + if setting not in self._settings: + raise InvalidSettingError('Setting {} is not valid'.format(setting)) + + sa = self._settings[setting] + setattr(self, setting, sa.conversion_fn(value)) + + def _get_settings(self): + """Retrieves the SettingAttributes for the configuration object. + + Returns: + A dictionary mapping of setting names to SettingAttributes. + """ + return { + 'workload_type': SettingAttribute( + validation_fn=is_valid_workload_type + ), + 'devices': SettingAttribute( + conversion_fn=try_split, + validation_fn=is_block_devices + ), + 'schedulers': SettingAttribute( + conversion_fn=try_split, + validation_fn=lambda x: validate_schedulers(x, self.devices), + dependent_attributes=['devices'] + ), + 'repetitions': SettingAttribute( + conversion_fn=int, + validation_fn=lambda x: x >= 1, + default_value=1 + ) + } + + +class FIOGlobalConfiguration(GlobalConfiguration): + pass +# endregion + + +# region Workload Configuration +class WorkloadConfiguration(ConfigSectionBase): + """Workload Configuration for `workload` sections of config. + + Args: + name: The name of the workload. + """ + def __init__(self, name): + super().__init__() + self._name = name + + def add_setting(self, setting, value): + """Adds a setting to the configuration object. + + Args: + setting: The setting. + value: The value. + + Raises: + InvalidSettingError: If setting is not defined in `_get_settings`. + """ + if setting not in self._settings: + raise InvalidSettingError('Setting {} is not valid'.format(setting)) + + sa = self._settings[setting] + setattr(self, setting, sa.conversion_fn(value)) + + def process(self, output_configuration, template_configuration, + global_configuration): + """Process the workload. + + Args: + output_configuration: The OutputConfiguration. + template_configuration: The TemplateConfiguration. + global_configuration: The GlobalConfiguration. + """ + printf('Processing workload {}'.format(self._name), + print_type=PrintType.INFO_LOG) + + devices = global_configuration.devices + schedulers = global_configuration.schedulers + repetitions = global_configuration.repetitions + job_type = get_job_type(global_configuration.workload_type) + + for device, scheduler in itertools.product(devices, schedulers): + for file, sp in template_configuration.get_file_permutations( + self.file, device, scheduler + ): + printf('Using template permutation {}'.format(sp), + print_type=PrintType.INFO_LOG) + + self.process_with_repetitions(output_configuration, file, device, + scheduler, job_type, repetitions, sp) + + def process_with_repetitions(self, output_configuration, file, device, + scheduler, job_type, repetitions, + setting_permutation): + """Process the workload for the template permutation and repetitions. + + Args: + output_configuration: The OutputConfiguration. + file: The input file. + device: The device to execute on. + scheduler: The schedulers to execute with. + job_type: The job type. + repetitions: The number of repetitions. + setting_permutation: The template setting permutation. + """ + for rep in range(repetitions): + printf('Executing file {} with device {}, scheduler {}, repetition ' + '{} of {}'.format(file, device, scheduler, rep + 1, repetitions), + print_type=PrintType.INFO_LOG) + + output = self._try_process(job_type, file, device, scheduler) + output_configuration.process(output, setting_permutation, + device, scheduler) + + def _try_process(self, job_type, file, device, scheduler): + """Attempts to process a job with retrying if failure. + + Args: + job_type: The job type. + file: The input file. + device: The device to execute on. + scheduler: The scheduler to execute with. + + Returns: + The output of processing the job. + + Raises: + RetryCountExceededError: If job fails and retry counts are exceeded. + """ + retry_count = SettingsManager.get('retry_count') + + for retry in range(retry_count): + if retry != 0: + printf('Retrying job...', print_type=PrintType.DEBUG_LOG) + + job = job_type(file, device, scheduler) + + try: + return job.process() + except JobExecutionError as err: + printf('Unable to run job \n{}'.format(err), + print_type=PrintType.ERROR_LOG) + + raise RetryCountExceededError( + 'Unable to run job, exceeded retry counts' + ) + + def _get_settings(self): + """Retrieves the SettingAttributes for the configuration object. + + Returns: + A dictionary mapping of setting names to SettingAttributes. + """ + return { + 'file': SettingAttribute() + } + + +class FIOWorkloadConfiguration(WorkloadConfiguration): + pass +# endregion + + +class TemplateConfiguration(ConfigSectionBase): + """Template Configuration for `template` section of config.""" + def __init__(self): + super().__init__() + self._dynamic_settings = set() + + def add_setting(self, setting, value): + """Adds a setting to the configuration object. + + Args: + setting: The setting. + value: The value. + """ + if setting in self._settings: + sa = self._settings[setting] + setattr(self, setting, sa.conversion_fn(value)) + + else: + setattr(self, setting, try_split(value, ',')) + self._dynamic_settings.add(setting) + + def get_file_permutations(self, file, device, scheduler): + """Creates interpolated files of permutated template settings. + + Args: + file: The input file. + device: The device. + scheduler: The scheduler. + + Returns: + Yields tuples of file names and setting permutations. + + Raises: + InvalidSettingError: If `file` does not exist. + """ + if not os.path.isfile(file): + raise InvalidSettingError( + 'Setting file {} does not exist'.format(file) + ) + + if self.enabled: + for sp in self._get_setting_permutations(): + temp_file_name = self._interpolate_file(file, device, scheduler, sp) + yield temp_file_name, sp + os.remove(temp_file_name) + else: + yield file, () + + def _get_setting_permutations(self): + """Retrieves setting permutations. + + Returns: + A list of setting permutation lists which are mappings of + `setting_name=setting_value`. + """ + setting_perm = [] + + for setting in self._dynamic_settings: + setting_perm.append([ + '{}={}'.format(setting, value) + for value in getattr(self, setting) + ]) + + return itertools.product(*setting_perm) + + def _interpolate_file(self, file, device, scheduler, sp): + """Creates a new file by reading and interpolating another. + + Args: + file: The input file. + device: The device. + scheduler: The scheduler. + sp: The permutated template settings. + + Returns: + The name of the new file. + """ + temp_file = file + '__temp__' + with open(file, 'r') as inp: + with open(temp_file, 'w') as out: + for line in inp: + int_line = self._interpolate_text( + line, device, scheduler, sp + ) + out.write(int_line) + + return temp_file + + def _interpolate_text(self, text, device, scheduler, sp): + """Interpolates text. + + Args: + text: The text to interpolate. + device: The device. + scheduler: The scheduler. + sp: The permutated template settings. + + Returns: + The interpolated text. + """ + tf = get_formatter('template') + device_name = match_regex(device, 'device_name') + + text = text.replace(tf.format('device'), device) + text = text.replace(tf.format('device_name'), device_name) + text = text.replace(tf.format('scheduler'), scheduler) + + for setting in sp: + name, value = setting.split('=') + text = text.replace(tf.format(name), value) + + return text + + def _get_settings(self): + """Retrieves the SettingAttributes for the configuration object. + + Returns: + A dictionary mapping of setting names to SettingAttributes. + """ + return { + 'enabled': SettingAttribute( + conversion_fn=cast_bool, + default_value=False + ) + } + + +# region Output Configuration +class OutputConfiguration(ConfigSectionBase): + """Output Configuration for `output` section of config. + + Args: + input_file: The input file. + """ + def __init__(self, input_file): + super().__init__() + self._input_file = input_file + self._wrote_header = False + + @abstractmethod + def _write_header(self, output, template_order, + setting_permutation_d, device, scheduler): + """Writes the header of the output file. + + Args: + output: The job output. + template_order: The ordered of setting permutations. + setting_permutation_d: The setting permutation in dict form. + device: The device. + scheduler: The scheduler. + """ + + @abstractmethod + def _write_line(self, output, template_order, + setting_permutation_d, device, scheduler): + """Writes a line of the output file. + + Args: + output: The job output. + template_order: The ordered of setting permutations. + setting_permutation_d: The setting permutation in dict form. + device: The device. + scheduler: The scheduler. + """ + + @abstractmethod + def _get_default_format(self): + """Retrieves the default format for the output if none is given. + + Returns: + A list of string. + """ + + def add_setting(self, setting, value): + """Adds a setting to the configuration object. + + Args: + setting: The setting. + value: The value. + + Raises: + InvalidSettingError: If setting is not defined in `_get_settings`. + """ + if setting not in self._settings: + raise InvalidSettingError('Setting {} is not valid'.format(setting)) + + sa = self._settings[setting] + setattr(self, setting, sa.conversion_fn(value)) + + def get_output_file(self): + """Retrieves the output file name. + + Returns: + The output file name. + """ + return self._input_file + '.csv' + + def process(self, output, setting_permutation, device, scheduler): + """Processes the output of a job. + + Args: + output: The job output. + setting_permutation: The template settings permutation. + device: The device. + scheduler: The scheduler. + """ + template_order = None + setting_permutation_d = None + + if self.append_template: + template_order = sorted([x.split('=')[0] for x in setting_permutation]) + spd = {} + for sp in setting_permutation: + k, v = sp.split('=') + spd[k] = v + + if not self._wrote_header: + self._write_header(output, template_order, setting_permutation_d, + device, scheduler) + self._wrote_header = True + + self._write_line(output, template_order, setting_permutation_d, + device, scheduler) + + def _get_settings(self): + """Retrieves the SettingAttributes for the configuration object. + + Returns: + A dictionary mapping of setting names to SettingAttributes. + """ + return { + 'format': SettingAttribute( + conversion_fn=try_split, + default_value=self._get_default_format() + ), + 'append_template': SettingAttribute( + conversion_fn=cast_bool, + default_value=True + ) + } + + def _get_universal_format_translation(self): + """Retrieves universal format translation. + + Returns: + A dictionary mapping formats to metrics. + """ + f = { + 'w': 'workload', + 'd': 'device', + 's': 'scheduler', + } + + f.update({x: x for x in f.values()}) + return f + + +class FIOOutputConfiguration(OutputConfiguration): + def _get_default_format(self): + """Retrieves the default format for the output if none is given. + + Returns: + A list of string. + """ + return [ + 'workload', + 'device', + 'scheduler', + 'job-runtime', + 'total-ios-read', + 'total-ios-write', + 'io-kbytes-read', + 'io-kbytes-write', + 'bw-read', + 'bw-write', + 'iops-read', + 'iops-write', + 'lat-min-read', + 'lat-min-write', + 'lat-max-read', + 'lat-max-write', + 'lat-mean-read', + 'lat-mean-write', + 'lat-stddev-read', + 'lat-stddev-write', + 'slat-min-read', + 'slat-min-write', + 'slat-max-read', + 'slat-max-write', + 'slat-mean-read', + 'slat-mean-write', + 'slat-stddev-read', + 'slat-stddev-write', + 'clat-min-read', + 'clat-min-write', + 'clat-max-read', + 'clat-max-write', + 'clat-mean-read', + 'clat-mean-write', + 'clat-stddev-read', + 'clat-stddev-write', + 'clat-percentiles-read', + 'clat-percentiles-write' + ] + + def _get_format_translation(self): + """Retrieves format translation. + + Returns: + A dictionary mapping formats to metrics. + """ + f = { + 'run': 'job-runtime', + 'tir': 'total-ios-read', + 'tiw': 'total-ios-write', + 'ibr': 'io-kbytes-read', + 'ibw': 'io-kbytes-write', + 'bwr': 'bw-read', + 'bww': 'bw-write', + 'opr': 'iops-read', + 'ipw': 'iops-write', + 'lir': 'lat-min-read', + 'liw': 'lat-min-write', + 'lar': 'lat-max-read', + 'law': 'lat-max-write', + 'lmr': 'lat-mean-read', + 'lmw': 'lat-mean-write', + 'lsr': 'lat-stddev-read', + 'lsw': 'lat-stddev-write', + 'sir': 'slat-min-read', + 'siw': 'slat-min-write', + 'sar': 'slat-max-read', + 'saw': 'slat-max-write', + 'smr': 'slat-mean-read', + 'smw': 'slat-mean-write', + 'ssr': 'slat-stddev-read', + 'ssw': 'slat-stddev-write', + 'cir': 'clat-min-read', + 'ciw': 'clat-min-write', + 'car': 'clat-max-read', + 'caw': 'clat-max-write', + 'cmr': 'clat-mean-read', + 'cmw': 'clat-mean-write', + 'csr': 'clat-stddev-read', + 'csw': 'clat-stddev-write' + } + + f.update({x: x for x in f.values()}) + return f + + def _get_clat_percentiles_format_translation(self): + """Retrieves percentiles format translation. + + Returns: + A dictionary mapping formats to metrics. + """ + f = { + 'cpr': 'clat-percentiles-read', + 'cpw': 'clat-percentiles-write', + } + + f.update({x: x for x in f.values()}) + return f + + def _get_lat_percentiles_format_translation(self): + """Retrieves percentiles format translation. + + Returns: + A dictionary mapping formats to metrics. + """ + f = { + 'lpr': 'lat-percentiles-read', + 'lpw': 'lat-percentiles-write' + } + + f.update({x: x for x in f.values()}) + return f + + def _get_percentiles_order(self, output): + """Returns a list of percentiles in ascending order. + + Args: + output: The job output. + + Returns: + A list of strings. + """ + return sorted([ + x for x in output if 'percentiles' in x + ], key=lambda x: int(x.split('-')[-1])) + + def _get_settings(self): + return { + **super()._get_settings(), + 'include_lat_percentiles': SettingAttribute( + conversion_fn=cast_bool, + default_value=False + ), + 'include_clat_percentiles': SettingAttribute( + conversion_fn=cast_bool, + default_value=False + ) + } + + def _write_header(self, output, template_order, setting_permutation_d, + device, scheduler): + """Writes the header of the output file. + + Args: + output: The job output. + template_order: The ordered of setting permutations. + setting_permutation_d: The setting permutation in dict form. + device: The device. + scheduler: The scheduler. + """ + output_file = self.get_output_file() + output_directory = SettingsManager.get('output_directory') + output_path = os.path.join(output_directory, output_file) + + ft = self._get_format_translation() + ut = self._get_universal_format_translation() + lpt = self._get_lat_percentiles_format_translation() + cpt = self._get_clat_percentiles_format_translation() + po = self._get_percentiles_order(output) + + first_line = True + with open(output_path, 'w+') as f: + for fi in self.format: + if not first_line: + f.write(',') + first_line = False + + if fi in ft: + f.write(ft[fi]) + elif fi in ut: + f.write(ut[fi]) + elif fi in lpt: + if self.include_lat_percentiles: + f.write(','.join(p for p in po if fi in p)) + elif fi in cpt: + if self.include_clat_percentiles: + f.write(','.join(p for p in po if fi in p)) + + else: + raise OutputFormatError( + 'Output format is invalid, unable to parse {}'.format(fi) + ) + + if self.append_template: + for t in template_order: + if not first_line: + f.write(',') + first_line = False + f.write(t) + + f.write('\n') + + def _write_line(self, output, template_order, setting_permutation_d, + device, scheduler): + """Writes a line of the output file. + + Args: + output: The job output. + template_order: The ordered of setting permutations. + setting_permutation_d: The setting permutation in dict form. + device: The device. + scheduler: The scheduler. + """ + output_file = self.get_output_file() + output_directory = SettingsManager.get('output_directory') + output_path = os.path.join(output_directory, output_file) + + ft = self._get_format_translation() + ut = self._get_universal_format_translation() + lpt = self._get_lat_percentiles_format_translation() + cpt = self._get_clat_percentiles_format_translation() + po = self._get_percentiles_order(output) + + first_line = True + with open(output_path, 'a') as f: + for fi in self.format: + if not first_line: + f.write(',') + first_line = False + + if fi in ft: + f.write(str(output[ft[fi]])) + elif fi in ut: + if fi == 'device': + f.write(device) + elif fi == 'scheduler': + f.write(scheduler) + elif fi in lpt: + if self.include_lat_percentiles: + f.write(','.join(output[p] for p in po if fi in p)) + elif fi in cpt: + if self.include_clat_percentiles: + f.write(','.join(output[p] for p in po if fi in p)) + else: + raise OutputFileError('Unable to write metric {}'.format(fi)) + + if self.append_template: + for t in template_order: + if not first_line: + f.write(',') + first_line = False + + f.write(str(setting_permutation_d[t])) + + f.write('\n') +# endregion + + +# region Jobs +class Job(ABC): + """A single unit of work to be executed. + + Args: + file: The input file. + device: The device. + scheduler: The scheduler. + """ + def __init__(self, file, device, scheduler): + self.file = file + self.device = device + self.scheduler = scheduler + + def process(self): + """Processes the job. + + Returns: + The output of the job. + """ + change_scheduler(self.device, self.scheduler) + clear_caches(self.device) + return self.execute() + + @abstractmethod + def execute(self): + """Executes the job.""" + + +class FIOJob(Job): + """An FIO Job.""" + def get_command(self): + """Retrieves the command to execute. + + Returns: + The command string. + """ + return 'fio {} --output-format=json'.format(self.file) + + def collect_output(self, output): + """Collects the output metrics from the job execution. + + Args: + output: The raw output. + + Returns: + A dictionary mapping metric names to values. + + Raises: + OutputParsingError: If unable to parse raw output. + """ + try: + data = json.loads(output, encoding='utf-8') + job_data = data['jobs'][0] + + metrics = { + **self._parse_job_other(job_data), + **self._parse_job_rw(job_data['read'], 'read'), + **self._parse_job_rw(job_data['write'], 'write') + } + + return metrics + except (json.JSONDecodeError, KeyError, IndexError) as err: + raise OutputParsingError( + 'Unable to parse output\n{}'.format(err) + ) + + def _parse_job_rw(self, data, rw): + """Parses the job data from the raw output. + + Args: + data: The data to parse. + rw: Either 'read' or 'write'. + + Returns: + A dictionary mapping the metric names to their values. + """ + metrics = { + 'total-ios-{}'.format(rw): data['total_ios'], # IO + 'io-kbytes-{}'.format(rw): data['io_kbytes'], # KB + 'bw-{}'.format(rw): data['bw'], # MB/s + 'iops-{}'.format(rw): data['iops'], # IO/s + 'lat-min-{}'.format(rw): data['lat_ns']['min'], # ns + 'lat-max-{}'.format(rw): data['lat_ns']['max'], # ns + 'lat-mean-{}'.format(rw): data['lat_ns']['mean'], # ns + 'lat-stddev-{}'.format(rw): data['lat_ns']['stddev'], # ns + 'slat-min-{}'.format(rw): data['slat_ns']['min'], # ns + 'slat-max-{}'.format(rw): data['slat_ns']['max'], # ns + 'slat-mean-{}'.format(rw): data['slat_ns']['mean'], # ns + 'slat-stddev-{}'.format(rw): data['slat_ns']['stddev'], # ns + 'clat-min-{}'.format(rw): data['clat_ns']['min'], # ns + 'clat-max-{}'.format(rw): data['clat_ns']['max'], # ns + 'clat-mean-{}'.format(rw): data['clat_ns']['mean'], # ns + 'clat-stddev-{}'.format(rw): data['clat_ns']['stddev'] # ns + } + + if 'percentile' in data['lat_ns']: + for p, v in data['lat_ns']['percentile'].items(): + metrics['lat-percentile-{}-{}'.format(p, rw)] = v # ns + + if 'percentile' in data['clat_ns']: + for p, v in data['clat_ns']['percentile'].items(): + metrics['clat-percentile-{}-{}'.format(p, rw)] = v # ns + + return metrics + + def _parse_job_other(self, data): + """Parses the other data from the raw output. + + Args: + data: The data to parse. + + Returns: + A dictionary mapping the metric names to their values. + """ + return { + 'job-runtime': data['job_runtime'] # ms + } + + def execute(self): + """Executes the job. + + Returns: + The collected output metrics. + + Raises JobExecutionError: If job failed to run. + """ + command = self.get_command() + out, _ = run_command(command) + + if out is None: + raise JobExecutionError( + 'Unable to run command {} for device {}' + .format(command, self.device) + ) + + printf('Job output:\n{}'.format(out), print_type=PrintType.DEBUG_LOG) + + return self.collect_output(out) +# endregion +# endregion diff --git a/iobs/errors.py b/iobs/errors.py new file mode 100644 index 0000000..000236d --- /dev/null +++ b/iobs/errors.py @@ -0,0 +1,87 @@ +# Copyright (c) 2018, UofL Computer Systems Lab. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without event the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +class IOBSBaseException(Exception): + """Base Exception for IOBS""" + + +class ConfigNotFoundError(IOBSBaseException): + """Config Not Found Error""" + + +class InvalidConfigError(IOBSBaseException): + """Invalid Config Error""" + + +class InvalidOSError(IOBSBaseException): + """Invalid OS Error""" + + +class InvalidPrivilegesError(IOBSBaseException): + """Invalid Privileges Error""" + + +class InvalidRegexError(IOBSBaseException): + """Invalid Regex Error""" + + +class InvalidSettingError(IOBSBaseException): + """Invalid Setting Error""" + + +class JobExecutionError(IOBSBaseException): + """Job Execution Error""" + + +class OutputFileError(IOBSBaseException): + """Output File Error""" + + +class OutputFormatError(IOBSBaseException): + """Output Format Error""" + + +class OutputParsingError(IOBSBaseException): + """Output Parsing Error""" + + +class RetryCountExceededError(IOBSBaseException): + """Retry Count Exceeded Error""" + + +class SchedulerChangeError(IOBSBaseException): + """Scheduler Change Error""" + + +class UndefinedConstantError(IOBSBaseException): + """Undefined Constant Error""" + + +class UndefinedFormatterError(IOBSBaseException): + """Undefined Formatter Error""" + + +class UndefinedRegexError(IOBSBaseException): + """Undefined Regex Error""" + + +class UndefinedWorkloadTypeError(IOBSBaseException): + """Undefined Workload Type Error""" + + +class UninitializedWorkloadConfigurationError(IOBSBaseException): + """Uninitialized Workload Configuration Error""" diff --git a/iobs/output.py b/iobs/output.py new file mode 100644 index 0000000..dd5de57 --- /dev/null +++ b/iobs/output.py @@ -0,0 +1,68 @@ +# Copyright (c) 2018, UofL Computer Systems Lab. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without event the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import enum +import logging + +from colorama import Fore + +from iobs.settings import SettingsManager + + +class PrintType(enum.IntEnum): + # STDOUT + NORMAL = 1 << 1 + WARNING = 1 << 2 + ERROR = 1 << 3 + + # LOG File + DEBUG_LOG = 1 << 4 + INFO_LOG = 1 << 5 + ERROR_LOG = 1 << 6 + + +def printf(*args, print_type=PrintType.NORMAL, **kwargs): + """Prints to STDOUT or log file depending on `print_type`. + + Args: + args: Arguments to pass to the print and/or log function. + print_type: Where and how to output the text. + kwargs: Keyword arguments to pass to the print and/or log function. + """ + args = [a.strip() if isinstance(a, str) else a for a in args] + silent = SettingsManager.get('silent') + log_enabled, log_level = SettingsManager.get('log_enabled', 'log_level') + + if not silent: + if print_type & PrintType.NORMAL: + print(*args, **kwargs) + + if print_type & PrintType.WARNING: + print(*[Fore.YELLOW + str(a) + Fore.RESET for a in args], **kwargs) + + if print_type & PrintType.ERROR: + print(*[Fore.RED + str(a) + Fore.RESET for a in args], **kwargs) + + if log_enabled: + if print_type & PrintType.ERROR_LOG: + logging.error(*args, **kwargs) + + if print_type & PrintType.INFO_LOG: + logging.info(*args, **kwargs) + + if print_type & PrintType.DEBUG_LOG: + logging.debug(*args, **kwargs) diff --git a/iobs/process.py b/iobs/process.py new file mode 100644 index 0000000..22dc3d9 --- /dev/null +++ b/iobs/process.py @@ -0,0 +1,485 @@ +# Copyright (c) 2018, UofL Computer Systems Lab. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without event the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +from collections import namedtuple +import os +import shlex +import signal +import stat +import subprocess + +from iobs.errors import SchedulerChangeError +from iobs.output import printf, PrintType +from iobs.settings import ( + match_regex +) + + +CommandProcess = namedtuple('CommandProcess', ('command', 'process')) + + +class ProcessManager: + PROCESSES = [] + + @staticmethod + def add_process(command, process): + """Tracks a running process. + + Args: + command: The command being run in the process. + process: The process. + """ + ProcessManager.PROCESSES += CommandProcess(command, process) + + @staticmethod + def clear_finished_processes(): + """Returns finished processes. + + Returns: + A list of finished CommandProcesses.""" + finished_processes = [] + process_index = 0 + + while process_index < len(ProcessManager.PROCESSES): + process = ProcessManager.PROCESSES[process_index] + if process[1].poll() in (None, 0): + ProcessManager.PROCESSES[process_index], ProcessManager.PROCESSES[-1] = \ + ProcessManager.PROCESSES[-1], ProcessManager.PROCESSES[process_index] + finished_processes.append(ProcessManager.PROCESSES[-1]) + ProcessManager.PROCESSES.pop() + else: + process_index += 1 + + return finished_processes + + @staticmethod + def clear_processes(): + """Clears all tracked processes.""" + ProcessManager.PROCESSES.clear() + + @staticmethod + def failed_processes(): + """Returns the processes which are failed. + + Returns: + A list of failed CommandProcesses. + """ + # Returns code other than 0 indicates error + return [p for p in ProcessManager.PROCESSES if p[1].poll() not in (None, 0)] + + @staticmethod + def finished_processes(): + """Returns the processes which are finished. + + Returns: + A list of finished CommandProcesses. + """ + # Return code 0 indicates success + return [p for p in ProcessManager.PROCESSES if p[1].poll() in (None, 0)] + + @staticmethod + def has_current_processes(): + """Returns whether there are any tracked processes currently running. + + Returns: + Number of running processes. + """ + return len(ProcessManager.PROCESSES) != 0 + + @staticmethod + def kill_processes(): + """Kills the processes.""" + printf('Killing running processes...', print_type=PrintType.DEBUG_LOG) + + for command_name, process in ProcessManager.PROCESSES: + try: + printf('Killing process %s [%s]' % (command_name, process.id), + print_type=PrintType.DEBUG_LOG) + os.killpg(os.getpgid(process.pid), signal.SIGTERM) + except Exception as err: + printf('Failed to kill process: %s [%s]\n%s' % (command_name, process.pid, err), + print_type=PrintType.ERROR_LOG) + + @staticmethod + def print_processes(): + """Prints the output of each process.""" + printf('Outputting running process information...', + print_type=PrintType.DEBUG_LOG) + + for command_name, process in ProcessManager.PROCESSES: + out, err = process.communicate() + if out: + printf('command {} output\n{}' + .format(command_name, out.decode('utf-8')), + print_type=PrintType.DEBUG_LOG) + + if err: + printf('command {} error\n{}' + .format(command_name, err.decode('utf-8')), + print_type=PrintType.DEBUG_LOG) + + +def change_scheduler(device, scheduler): + """Changes the I/O scheduler for the given device. + + Args: + device: The device. + scheduler: The I/O scheduler. + + Returns: + True if successful, else False. + """ + printf('Changing scheduler for device {} to {}'.format(device, scheduler), + print_type=PrintType.DEBUG_LOG) + + command = 'bash -c "echo {} > /sys/block/{}/queue/scheduler"' \ + .format(scheduler, match_regex(device, 'device_name')) + + _, rc = run_command(command) + + if rc != 0: + raise SchedulerChangeError( + 'Unable to change scheduler {} for device {}'.format(scheduler, device) + ) + + +def check_command(command): + """Returns whether the given command exists on the system. + + Args: + command: The command. + + Returns: + True if exists, else False. + """ + printf('Checking if command {} exists'.format(command), + print_type=PrintType.DEBUG_LOG) + + if run_system_command('command -v {}'.format(command)) == 0: + return True + + printf('Command {} does not exist'.format(command), + print_type=PrintType.ERROR_LOG) + + return False + + +def check_commands(commands): + """Checks whether the required utilities are available on the system. + + Returns: + True if all commands are valid, else False. + """ + return all(check_command(c) for c in commands) + + +def clear_caches(device): + """Clears various data caches. Should be run before each benchmark. + + Args: + device: The device. + """ + printf('Clearing caches for device {}'.format(device), + print_type=PrintType.DEBUG_LOG) + + # Writes any data buffered in memory out to disk + run_system_command('sync') + + # Drops clean caches + run_system_command('echo 3 > /proc/sys/vm/drop_caches') + + # Calls block device ioctls to flush buffers + run_system_command('blockdev --flushbufs {}'.format(device)) + + # Flushes the on-drive write cache buffer + run_system_command('hdparm -F {}'.format(device)) + + +def cleanup_files(files): + """Removes the specified file, or files if multiple are given. + + Args: + files: The files to remove. + """ + printf('Cleaning up files', print_type=PrintType.DEBUG_LOG) + + for file in files: + printf('Removing files %s' % file, print_type=PrintType.DEBUG_LOG) + if run_system_command('rm -f {}'.format(file)) != 0: + printf('Unable to clean up files: {}'.format(file), + print_type=PrintType.ERROR_LOG) + + +def get_device_major_minor(device): + """Returns a string of the major, minor of a given device. + + Args: + device: The device. + + Returns: + A string of major,minor. + """ + printf('Retrieving major,minor for device {}'.format(device), + print_type=PrintType.DEBUG_LOG) + + out, _ = run_command('stat -c \'%%t,%%T\' {}'.format(device)) + + if not out: + printf('Unable to retrieve major,minor information for device {}' + .format(device), + print_Type=PrintType.ERROR_LOG) + return None + + out = out.strip() + printf('major,minor for device {} is {}'.format(device, out), + print_Type=PrintType.DEBUG_LOG) + + return out + + +def get_schedulers_for_device(device): + """Returns a list of available schedulers for a given device. + + Args: + device: The device. + + Returns: + A list of schedulers. + """ + printf('Retrieving schedulers for device {}'.format(device), + print_type=PrintType.DEBUG_LOG) + + device_name = match_regex(device, 'device_name') + + out, rc = run_command('cat /sys/block/{}/queue/scheduler'.format(device_name)) + + if rc != 0: + printf('Unable to find schedulers for device', + print_type=PrintType.ERROR_LOG) + return [] + + ret = out.strip().replace('[', '').replace(']', '') + + printf('Found the following schedulers for device {}: ' + '{}'.format(device, ret), + print_type=PrintType.DEBUG_LOG) + + return ret.split() + + +def is_block_device(device): + """Returns whether the given device is a valid block device. + + Args: + device: The device. + + Returns: + True if is a valid block device, else False. + """ + printf('Checking if device {} is a valid block device'.format(device), + print_type=PrintType.DEBUG_LOG) + + try: + if stat.S_ISBLK(os.stat(device).st_mode): + printf('Device {} is a valid block device'.format(device), + print_type=PrintType.DEBUG_LOG) + return True + + printf('Device {} is not a valid block device'.format(device), + print_type=PrintType.ERROR_LOG) + return False + except (FileNotFoundError, TypeError): + printf('Device {} is not a valid block device'.format(device), + print_type=PrintType.ERROR_LOG) + return False + + +def is_block_devices(devices): + """Returns whether the given devices are valid block devices. + + Args: + devices: The devices. + + Returns: + True if all are valid block device, else False. + """ + return all(is_block_device(device) for device in devices) + + +def is_rotational_device(device): + """Returns whether the given device is a rotational device. + + Args: + device: The device. + + Returns: + True if is a rotational device, else False. + """ + printf('Checking whether device {} is a rotational device'.format(device), + print_type=PrintType.DEBUG_LOG) + + device_name = match_regex(device, 'device_name') + + if not device_name: + return False + + out, rc = run_command( + 'cat /sys/block/{}/queue/rotational'.format(device_name) + ) + + if rc != 0: + return False + + if int(out) == 1: + printf('Device {} is a rotational device'.format(device), + print_type=PrintType.DEBUG_LOG) + else: + printf('Device {} is not a rotational device'.format(device), + print_type=PrintType.DEBUG_LOG) + + return int(out) == 1 + + +def run_command(command, ignore_output=False): + """Runs a command via subprocess communication. + + Args: + command: The command. + ignore_output: (OPTIONAL) Whether to ignore the output. Defaults to + False. + + Returns: + A tuple containing (the output, the return code). + """ + printf('Running command {}'.format(command), print_type=PrintType.DEBUG_LOG) + + try: + args = shlex.split(command) + p = subprocess.Popen( + args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + stdin=subprocess.PIPE, + preexec_fn=os.setsid) + + ProcessManager.add_process(command, p) + + if ignore_output: + return None, _wait_for_process(command, p) + + return _communicate_to_process(command, p) + except (ValueError, subprocess.CalledProcessError, FileNotFoundError) as err: + printf('Command {} erred:\n{}'.format(command, err), + print_type=PrintType.ERROR_LOG) + return None, None + finally: + ProcessManager.clear_processes() + + +def _wait_for_process(command, p): + """Waits for a process to complete. + + Args: + command: The command. + p: The process. + + Returns: + The return code. + """ + rc = p.wait() + + if rc != 0: + printf('Command {} [{}] erred with return code {}' + .format(command, p.pid, rc), + print_type=PrintType.ERROR_LOG) + return rc + + +def _communicate_to_process(command, p): + """Communicates to a process. + + Args: + command: The command. + p: The process. + + Returns: + A tuple of the output and return code. + """ + out, err = p.communicate() + rc = p.returncode + + if err: + printf('Command {} [{}] erred with return code {}:\n{}' + .format(command, p.pid, rc, err.decode('utf-8')), + print_type=PrintType.ERROR_LOG) + + return out.decode('utf-8'), rc + + +def run_system_command(command, silence=True): + """Runs a system command. + + Args: + command: The command. + silence: (OPTIONAL) Whether to silence the console output. Defaults to + True. + + Returns: + The return code. + """ + if silence: + command = '{} >/dev/null 2>&1'.format(command) + + printf('Running command {}'.format(command), + print_type=PrintType.DEBUG_LOG) + + try: + return os.system(command) + except Exception as err: + printf('Error occurred running command {}\n{}'.format(command, err), + print_type=PrintType.ERROR_LOG) + return -1 + + +def validate_schedulers(schedulers, devices): + """Validates all schedulers are available on the devices. + + Args: + schedulers: The schedulers. + devices: The devices. + + Returns: + True if all are valid, else False. + """ + return all( + validate_schedulers_for_device(schedulers, device) + for device in devices + ) + + +def validate_schedulers_for_device(schedulers, device): + """Validates the schedulers are available on the device. + + Args: + schedulers: The schedulers. + device: The device. + + Returns: + True if all are valid, else False. + """ + valid_schedulers = set(get_schedulers_for_device(device)) + return all(sched in valid_schedulers for sched in schedulers) diff --git a/iobs/settings.py b/iobs/settings.py new file mode 100644 index 0000000..48b198c --- /dev/null +++ b/iobs/settings.py @@ -0,0 +1,255 @@ +# Copyright (c) 2018, UofL Computer Systems Lab. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without event the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import os +import re + +from iobs.errors import ( + InvalidSettingError, + UndefinedConstantError, + UndefinedFormatterError, + UndefinedRegexError +) + + +_CONSTANTS = { + 'config_header_global': 'global', + 'config_header_output': 'output', + 'config_header_template': 'template', + 'valid_workload_types': {'fio'} +} + +_FORMATTERS = { + 'template': '<%{}%>' +} + + +_REGEX = { + 'device_name': re.compile(r'/dev/(.*)') +} + + +class SettingsManager: + """Controls settings set by command-line arguments.""" + @staticmethod + def get(*settings): + """Retrieves attributes on self. + + Args: + settings: The attributes to retrieve. + + Returns: + The values of the attributes on self. + + Raises: + InvalidSettingError: If setting does not exist on self. + """ + ret = [] + for setting in settings: + try: + ret.append(getattr(SettingsManager, setting)) + except AttributeError: + raise InvalidSettingError('{} does not exist'.format(setting)) + + if len(ret) == 1: + return ret[0] + return ret + + @staticmethod + def set(setting, value): + """Sets an attribute on self. + + Args: + setting: The attribute to set. + value: The value to set the attribute to. + """ + setattr(SettingsManager, setting, value) + + +class SettingAttribute: + """Attribute properties for settings. + + Args: + conversion_fn: Function to convert the string representation into + another type. + validation_fn: Function to validate the value of the setting. + dependent_attributes: Other attributes which this is dependent on. + default_value: Default value if none explicitly assigned. + """ + def __init__(self, conversion_fn=lambda x: str(x), + validation_fn=lambda x: True, + dependent_attributes=None, + default_value=None): + self.conversion_fn = conversion_fn + self.validation_fn = validation_fn + self.dependent_attributes = dependent_attributes + self.default_value = default_value + + +def get_constant(name): + """Retrieves a constant. + + Args: + name: The name of the constant. + + Returns: + The constant. + + Raises: + UndefinedConstantError: If constant is not defined. + """ + if name not in _CONSTANTS: + raise UndefinedConstantError( + 'Constant {} is not defined'.format(name) + ) + + return _CONSTANTS[name] + + +def get_formatter(name): + """Retrieves a formatter. + + Args: + name: The name of the formatter. + + Returns: + The formatter. + + Raises: + UndefinedFormatterError: If formatter is not defined. + """ + if name not in _FORMATTERS: + raise UndefinedFormatterError( + 'Formatter {} is not defined'.format(name) + ) + + return _FORMATTERS[name] + + +def is_valid_workload_type(workload_type): + """Validates a given workload type. + + Args: + workload_type: The workload type. + + Returns: + True if value, else False. + """ + return workload_type in get_constant('valid_workload_types') + + +def match_regex(string, regex_name): + """Returns the matching regex pattern in the string. + + Args: + string: The string to search. + regex_name: The name of the regex to match on. + + Returns: + Regex match or None if there isn't a match. + + Raises: + UndefinedRegexError: If `regex_name` isn't a defined regex. + """ + if regex_name not in _REGEX: + raise UndefinedRegexError('regex {} is not defined'.format(regex_name)) + + regex = _REGEX[regex_name] + match = regex.match(string) + + if not match: + return None + + return match[1] + + +def validate_setting(obj, setting_name, setting): + """Validates the setting attribute on an object. + + Args: + obj: The object. + setting_name: The attribute. + setting: The SettingAttribute. + + Raises: + InvalidSettingError: If no value set and `default_value` not set on + `setting`. Or if fails `validate_fn` on `setting`. + """ + setting_value = getattr(obj, setting_name, None) + if setting_value is None: + if setting.default_value is None: + raise InvalidSettingError( + 'Required setting {} is not defined'.format(setting_name) + ) + + setting_value = setting.default_value + setattr(obj, setting_name, setting.default_value) + + if not setting.validation_fn(setting_value): + raise InvalidSettingError( + 'Setting {}={} is not valid'.format(setting_name, setting_value) + ) + + +def validate_settings(obj, settings): + """Validates the settings on an object. + + Args: + obj: An object to check attributes of. + settings: A dictionary mapping names to SettingAttributes. + + Raises: + InvalidSettingError: If settings are not valid or dependencies not met. + """ + # NOTE: Assumes that there are no circular dependencies + roots = { + k for k, v in settings.items() + if not v.dependent_attributes + } + unvalidated = { + k for k, v in settings.items() + if v.dependent_attributes + } + inc_deps = {s: set() for s in settings} + out_deps = { + k: set(v.dependent_attributes) + for k, v in settings.items() + if v.dependent_attributes + } + + for k, v in settings.items(): + if v.dependent_attributes: + for dep in v.dependent_attributes: + inc_deps[dep].add(k) + + while roots: + setting_name = roots.pop() + setting = settings[setting_name] + validate_setting(obj, setting_name, setting) + + for dep in inc_deps[setting_name]: + out_deps[dep].remove(setting_name) + if not out_deps[dep]: + del out_deps[dep] + unvalidated.remove(dep) + roots.add(dep) + + if unvalidated: + raise InvalidSettingError( + 'Setting(s) {} do not have dependencies met' + .format(', '.join(unvalidated)) + ) diff --git a/iobs/util.py b/iobs/util.py new file mode 100644 index 0000000..f0d7f2a --- /dev/null +++ b/iobs/util.py @@ -0,0 +1,84 @@ +# Copyright (c) 2018, UofL Computer Systems Lab. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without event the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +from functools import wraps + + +def cast_bool(obj): + """Attempts to cast an object as a bool using the following rules: + 1. If int: True if 1, else False. + 2. If str: True if lower-cased is 't' or 'true', else False. + 3. bool(obj) + + Args: + obj: The object to cast. + + Returns: + Boolean representation of object. + """ + if isinstance(obj, int): + return obj == 1 + + if isinstance(obj, str): + return obj.lower() in ('t', 'true', '1') + + return bool(obj) + + +def ignore_exception(exception, default_val): + """A decorator function. + + A decorator function that ignores the exception raised, and instead returns + a default value. + + Args: + exception: The exception to catch. + default_val: The decorated function. + + Returns: + The decorator function. + """ + def decorator(func): + @wraps(func) + def wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except exception: + return default_val + return wrapper + return decorator + + +def try_split(s, delimiter=','): + """Tries to split a string by the given delimiter(s). + + Args: + s: The string to split. + delimiter: Either a single string, or a tuple of strings + (i.e. (',', ';'). + + Returns: + The string split into a list. + """ + if isinstance(delimiter, tuple): + for d in delimiter: + if d in s: + return [i.strip() for i in s.split(d)] + elif delimiter in s: + return s.split(delimiter) + + return [s] From 03476c2126b9df50d472185e5d5dc45772a63cca Mon Sep 17 00:00:00 2001 From: Jared Gillespie Date: Mon, 25 Feb 2019 17:08:09 -0500 Subject: [PATCH 08/21] Adding output directory arg --- iobs/commands/execute.py | 12 ++++++------ iobs/config.py | 6 ++++-- iobs/settings.py | 6 ++++++ 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/iobs/commands/execute.py b/iobs/commands/execute.py index aba47e0..0efb4d1 100644 --- a/iobs/commands/execute.py +++ b/iobs/commands/execute.py @@ -43,7 +43,7 @@ def check_args(args): if args.log_file: SettingsManager.set('log_enabled', True) log_level = get_log_level(args.log_level) - log_path = os.path.join(os.getcwd(), args.log_file) + log_path = os.path.join(args.log_file) logging.basicConfig(filename=log_path, level=log_level, format='%(asctime)s - %(message)s') else: @@ -105,13 +105,13 @@ def execute(args): Raises: IOBSBaseException: If error occurs and `continue_on_failure` not set. """ - printf('Beginning program execution...', - print_type=PrintType.NORMAL | PrintType.INFO_LOG) - check_args(args) validate_os() validate_privileges() + printf('Beginning program execution...', + print_type=PrintType.NORMAL | PrintType.INFO_LOG) + for i, input_file in enumerate(args.inputs): try: printf('Processing input file {} ({} of {})' @@ -146,8 +146,8 @@ def main(args): '-o', '--output-directory', dest='output_directory', default=os.getcwd(), - help='The output directory for output and log files. Defaults to the ' - 'current working directory.' + help='The output directory for output files. Defaults to the current ' + 'working directory.' ) parser.add_argument( '-l', '--log-file', diff --git a/iobs/config.py b/iobs/config.py index ddc9ec4..2e4c3f0 100644 --- a/iobs/config.py +++ b/iobs/config.py @@ -874,7 +874,8 @@ def _write_header(self, output, template_order, setting_permutation_d, device: The device. scheduler: The scheduler. """ - output_file = self.get_output_file() + output_base = os.path.basename(self.get_output_file()) + output_file = os.path.splitext(output_base)[0] output_directory = SettingsManager.get('output_directory') output_path = os.path.join(output_directory, output_file) @@ -927,7 +928,8 @@ def _write_line(self, output, template_order, setting_permutation_d, device: The device. scheduler: The scheduler. """ - output_file = self.get_output_file() + output_base = os.path.basename(self.get_output_file()) + output_file = os.path.splitext(output_base)[0] output_directory = SettingsManager.get('output_directory') output_path = os.path.join(output_directory, output_file) diff --git a/iobs/settings.py b/iobs/settings.py index 48b198c..46bdc77 100644 --- a/iobs/settings.py +++ b/iobs/settings.py @@ -45,6 +45,12 @@ class SettingsManager: """Controls settings set by command-line arguments.""" + continue_on_failure = False + log_enabled = False + output_directory = os.getcwd() + retry_count = 1 + silent = False + @staticmethod def get(*settings): """Retrieves attributes on self. From 7c3fc170c71298afba7548e297d2909d053ded51 Mon Sep 17 00:00:00 2001 From: Jared Gillespie Date: Mon, 25 Feb 2019 17:08:29 -0500 Subject: [PATCH 09/21] Fixing config --- iobs/config.py | 68 ++++++++++++++++++++++++++------------------------ 1 file changed, 36 insertions(+), 32 deletions(-) diff --git a/iobs/config.py b/iobs/config.py index 2e4c3f0..4d7753a 100644 --- a/iobs/config.py +++ b/iobs/config.py @@ -408,7 +408,7 @@ def process_with_repetitions(self, output_configuration, file, device, output = self._try_process(job_type, file, device, scheduler) output_configuration.process(output, setting_permutation, - device, scheduler) + self._name, device, scheduler) def _try_process(self, job_type, file, device, scheduler): """Attempts to process a job with retrying if failure. @@ -600,26 +600,28 @@ def __init__(self, input_file): @abstractmethod def _write_header(self, output, template_order, - setting_permutation_d, device, scheduler): + setting_permutation_d, workload, device, scheduler): """Writes the header of the output file. Args: output: The job output. template_order: The ordered of setting permutations. setting_permutation_d: The setting permutation in dict form. + workload: The workload name. device: The device. scheduler: The scheduler. """ @abstractmethod def _write_line(self, output, template_order, - setting_permutation_d, device, scheduler): + setting_permutation_d, workload, device, scheduler): """Writes a line of the output file. Args: output: The job output. template_order: The ordered of setting permutations. setting_permutation_d: The setting permutation in dict form. + workload: The workload name. device: The device. scheduler: The scheduler. """ @@ -656,12 +658,13 @@ def get_output_file(self): """ return self._input_file + '.csv' - def process(self, output, setting_permutation, device, scheduler): + def process(self, output, setting_permutation, workload, device, scheduler): """Processes the output of a job. Args: output: The job output. setting_permutation: The template settings permutation. + workload: The workload name. device: The device. scheduler: The scheduler. """ @@ -670,18 +673,18 @@ def process(self, output, setting_permutation, device, scheduler): if self.append_template: template_order = sorted([x.split('=')[0] for x in setting_permutation]) - spd = {} + setting_permutation_d = {} for sp in setting_permutation: k, v = sp.split('=') - spd[k] = v + setting_permutation_d[k] = v if not self._wrote_header: self._write_header(output, template_order, setting_permutation_d, - device, scheduler) + workload, device, scheduler) self._wrote_header = True self._write_line(output, template_order, setting_permutation_d, - device, scheduler) + workload, device, scheduler) def _get_settings(self): """Retrieves the SettingAttributes for the configuration object. @@ -864,13 +867,14 @@ def _get_settings(self): } def _write_header(self, output, template_order, setting_permutation_d, - device, scheduler): + workload, device, scheduler): """Writes the header of the output file. Args: output: The job output. template_order: The ordered of setting permutations. setting_permutation_d: The setting permutation in dict form. + workload: The workload name. device: The device. scheduler: The scheduler. """ @@ -885,23 +889,22 @@ def _write_header(self, output, template_order, setting_permutation_d, cpt = self._get_clat_percentiles_format_translation() po = self._get_percentiles_order(output) - first_line = True with open(output_path, 'w+') as f: for fi in self.format: - if not first_line: - f.write(',') - first_line = False - if fi in ft: f.write(ft[fi]) + f.write(',') elif fi in ut: f.write(ut[fi]) + f.write(',') elif fi in lpt: if self.include_lat_percentiles: f.write(','.join(p for p in po if fi in p)) + f.write(',') elif fi in cpt: if self.include_clat_percentiles: f.write(','.join(p for p in po if fi in p)) + f.write(',') else: raise OutputFormatError( @@ -910,21 +913,20 @@ def _write_header(self, output, template_order, setting_permutation_d, if self.append_template: for t in template_order: - if not first_line: - f.write(',') - first_line = False f.write(t) + f.write(',') - f.write('\n') + f.write('END\n') def _write_line(self, output, template_order, setting_permutation_d, - device, scheduler): + workload, device, scheduler): """Writes a line of the output file. Args: output: The job output. template_order: The ordered of setting permutations. setting_permutation_d: The setting permutation in dict form. + workload: The workload name. device: The device. scheduler: The scheduler. """ @@ -939,38 +941,40 @@ def _write_line(self, output, template_order, setting_permutation_d, cpt = self._get_clat_percentiles_format_translation() po = self._get_percentiles_order(output) - first_line = True with open(output_path, 'a') as f: for fi in self.format: - if not first_line: - f.write(',') - first_line = False - if fi in ft: f.write(str(output[ft[fi]])) + f.write(',') elif fi in ut: - if fi == 'device': + if fi == 'workload': + f.write(workload) + elif fi == 'device': f.write(device) elif fi == 'scheduler': f.write(scheduler) + else: + raise OutputFormatError( + 'Unable to write metric {}'.format(fi) + ) + f.write(',') elif fi in lpt: if self.include_lat_percentiles: - f.write(','.join(output[p] for p in po if fi in p)) + f.write(','.join(str(output[p]) for p in po if fi in p)) + f.write(',') elif fi in cpt: if self.include_clat_percentiles: - f.write(','.join(output[p] for p in po if fi in p)) + f.write(','.join(str(output[p]) for p in po if fi in p)) + f.write(',') else: raise OutputFileError('Unable to write metric {}'.format(fi)) if self.append_template: for t in template_order: - if not first_line: - f.write(',') - first_line = False - f.write(str(setting_permutation_d[t])) + f.write(',') - f.write('\n') + f.write('END\n') # endregion From 498a76cb91b8b4180ea56e2e2f8a177c556adeaf Mon Sep 17 00:00:00 2001 From: Jared Gillespie Date: Mon, 25 Feb 2019 17:08:43 -0500 Subject: [PATCH 10/21] Removing log_level from output --- iobs/output.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iobs/output.py b/iobs/output.py index dd5de57..8a09d7b 100644 --- a/iobs/output.py +++ b/iobs/output.py @@ -45,7 +45,7 @@ def printf(*args, print_type=PrintType.NORMAL, **kwargs): """ args = [a.strip() if isinstance(a, str) else a for a in args] silent = SettingsManager.get('silent') - log_enabled, log_level = SettingsManager.get('log_enabled', 'log_level') + log_enabled = SettingsManager.get('log_enabled') if not silent: if print_type & PrintType.NORMAL: From 38b6b9e6f70349ef359c6d3f86bcdfeffaf57921 Mon Sep 17 00:00:00 2001 From: Jared Gillespie Date: Mon, 25 Feb 2019 17:36:59 -0500 Subject: [PATCH 11/21] Prefixing log errors with ERROR --- iobs/output.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iobs/output.py b/iobs/output.py index 8a09d7b..8fadd92 100644 --- a/iobs/output.py +++ b/iobs/output.py @@ -59,7 +59,7 @@ def printf(*args, print_type=PrintType.NORMAL, **kwargs): if log_enabled: if print_type & PrintType.ERROR_LOG: - logging.error(*args, **kwargs) + logging.error('ERROR: ' + args[0], *args[1:], **kwargs) if print_type & PrintType.INFO_LOG: logging.info(*args, **kwargs) From 68ffd939c79c3ac245154fe7894eabf3242bb03f Mon Sep 17 00:00:00 2001 From: Jared Gillespie Date: Mon, 25 Feb 2019 22:01:05 -0500 Subject: [PATCH 12/21] Fixing percentile output --- iobs/config.py | 76 ++++++++++++++++++++++++++++++-------------------- 1 file changed, 46 insertions(+), 30 deletions(-) diff --git a/iobs/config.py b/iobs/config.py index 4d7753a..ae49499 100644 --- a/iobs/config.py +++ b/iobs/config.py @@ -763,8 +763,8 @@ def _get_default_format(self): 'clat-mean-write', 'clat-stddev-read', 'clat-stddev-write', - 'clat-percentiles-read', - 'clat-percentiles-write' + 'clat-percentile-read', + 'clat-percentile-write' ] def _get_format_translation(self): @@ -812,36 +812,36 @@ def _get_format_translation(self): f.update({x: x for x in f.values()}) return f - def _get_clat_percentiles_format_translation(self): - """Retrieves percentiles format translation. + def _get_clat_percentile_format_translation(self): + """Retrieves percentile format translation. Returns: A dictionary mapping formats to metrics. """ f = { - 'cpr': 'clat-percentiles-read', - 'cpw': 'clat-percentiles-write', + 'cpr': 'clat-percentile-read', + 'cpw': 'clat-percentile-write', } f.update({x: x for x in f.values()}) return f - def _get_lat_percentiles_format_translation(self): - """Retrieves percentiles format translation. + def _get_lat_percentile_format_translation(self): + """Retrieves percentile format translation. Returns: A dictionary mapping formats to metrics. """ f = { - 'lpr': 'lat-percentiles-read', - 'lpw': 'lat-percentiles-write' + 'lpr': 'lat-percentile-read', + 'lpw': 'lat-percentile-write' } f.update({x: x for x in f.values()}) return f - def _get_percentiles_order(self, output): - """Returns a list of percentiles in ascending order. + def _get_percentile_order(self, output): + """Returns a list of percentile in ascending order. Args: output: The job output. @@ -850,22 +850,26 @@ def _get_percentiles_order(self, output): A list of strings. """ return sorted([ - x for x in output if 'percentiles' in x - ], key=lambda x: int(x.split('-')[-1])) + x for x in output if 'percentile' in x + ], key=lambda x: float(x.split('-')[-2])) def _get_settings(self): return { **super()._get_settings(), - 'include_lat_percentiles': SettingAttribute( + 'include_lat_percentile': SettingAttribute( conversion_fn=cast_bool, default_value=False ), - 'include_clat_percentiles': SettingAttribute( + 'include_clat_percentile': SettingAttribute( conversion_fn=cast_bool, default_value=False ) } + def _compare_percentile_format(self, setting_name, percentile_metric): + pms = percentile_metric.split('-') + return setting_name == '-'.join([pms[0], pms[1], pms[3]]) + def _write_header(self, output, template_order, setting_permutation_d, workload, device, scheduler): """Writes the header of the output file. @@ -885,9 +889,9 @@ def _write_header(self, output, template_order, setting_permutation_d, ft = self._get_format_translation() ut = self._get_universal_format_translation() - lpt = self._get_lat_percentiles_format_translation() - cpt = self._get_clat_percentiles_format_translation() - po = self._get_percentiles_order(output) + lpt = self._get_lat_percentile_format_translation() + cpt = self._get_clat_percentile_format_translation() + po = self._get_percentile_order(output) with open(output_path, 'w+') as f: for fi in self.format: @@ -898,12 +902,18 @@ def _write_header(self, output, template_order, setting_permutation_d, f.write(ut[fi]) f.write(',') elif fi in lpt: - if self.include_lat_percentiles: - f.write(','.join(p for p in po if fi in p)) + if self.include_lat_percentile: + f.write(','.join( + p for p in po + if self._compare_percentile_format(fi, p)) + ) f.write(',') elif fi in cpt: - if self.include_clat_percentiles: - f.write(','.join(p for p in po if fi in p)) + if self.include_clat_percentile: + f.write(','.join( + p for p in po + if self._compare_percentile_format(fi, p)) + ) f.write(',') else: @@ -937,9 +947,9 @@ def _write_line(self, output, template_order, setting_permutation_d, ft = self._get_format_translation() ut = self._get_universal_format_translation() - lpt = self._get_lat_percentiles_format_translation() - cpt = self._get_clat_percentiles_format_translation() - po = self._get_percentiles_order(output) + lpt = self._get_lat_percentile_format_translation() + cpt = self._get_clat_percentile_format_translation() + po = self._get_percentile_order(output) with open(output_path, 'a') as f: for fi in self.format: @@ -959,12 +969,18 @@ def _write_line(self, output, template_order, setting_permutation_d, ) f.write(',') elif fi in lpt: - if self.include_lat_percentiles: - f.write(','.join(str(output[p]) for p in po if fi in p)) + if self.include_lat_percentile: + f.write(','.join(str( + output[p]) for p in po + if self._compare_percentile_format(fi, p)) + ) f.write(',') elif fi in cpt: - if self.include_clat_percentiles: - f.write(','.join(str(output[p]) for p in po if fi in p)) + if self.include_clat_percentile: + f.write(','.join(str( + output[p]) for p in po + if self._compare_percentile_format(fi, p)) + ) f.write(',') else: raise OutputFileError('Unable to write metric {}'.format(fi)) From e21e67d23e5ab7f132ed656e8c1eca2dd0bcf9b1 Mon Sep 17 00:00:00 2001 From: Jared Gillespie Date: Tue, 26 Feb 2019 10:18:51 -0500 Subject: [PATCH 13/21] MANIFEST.in recursive include examples --- MANIFEST.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 165ece1..e140edd 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,4 +3,4 @@ include CHANGELOG.md include LICENSE include AUTHORS -recursive-include examples +recursive-include examples * From b6ff243f7544f0fe79875f74682427dd5241bd32 Mon Sep 17 00:00:00 2001 From: Jared Gillespie Date: Tue, 26 Feb 2019 10:41:19 -0500 Subject: [PATCH 14/21] Spacing in log-level arg --- iobs/commands/execute.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iobs/commands/execute.py b/iobs/commands/execute.py index 0efb4d1..98d8a82 100644 --- a/iobs/commands/execute.py +++ b/iobs/commands/execute.py @@ -161,7 +161,7 @@ def main(args): default=1, type=int, help='The level of information to which to log: 1 (Debug), ' - '2 (Info), 3(Error). Defaults to 2.' + '2 (Info), 3 (Error). Defaults to 2.' ) parser.add_argument( '-s', '--silent', From 30c7e0f4d984750f678848baa25829040ce30064 Mon Sep 17 00:00:00 2001 From: Jared Gillespie Date: Tue, 26 Feb 2019 16:26:09 -0500 Subject: [PATCH 15/21] Update README.md --- README.md | 322 +++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 244 insertions(+), 78 deletions(-) diff --git a/README.md b/README.md index d2a6349..1823c0e 100644 --- a/README.md +++ b/README.md @@ -1,90 +1,256 @@ # Linux I/O Benchmark for Schedulers (iobs) -An I/O workload automation and metric analysis tool used to characterize different workloads for different configurations. The following Linux tools are utilized internally: `blktrace`, `blkparse`, `btt`, `blkrawverify`, and (optionally) `fio`. + +An I/O workload automation and metric analysis tool used to gauge the performance of a device. It provides a means of +automating commonly run workloads with tools such as `fio`. + +## Why Should I Use This? + +The goal of `iobs` is to decrease the amount of manual work involved in running I/O experiments on devices. + +Commonly used tools for running workloads, such as `fio` are well suited for providing a means of benchmarking a device. +However, when utilizing other tools or running multiple workloads with slight variations, the number of configuration +changes and formatting differences between tools makes for an inefficient amount of manual work required to consolidate +the results. + +The biggest advantage of `iobs` is the reduction in work required to run many experiments with different combinations +of configurations. ## Installation -The latest version can be obtained via `wget`: -```bash -$ wget https://raw.githubusercontent.com/UOFL-CSL/iobs/master/iobs.py -``` + +The latest version can be obtained from the [releases](https://github.com/uofl-csl/iobs/releases). + +The following steps are recommended in retrieving the package: + 1. Use `wget` to pull the latest [tarball](https://github.com/uofl-csl/iobs/releases) onto the machine. + 2. Run `tar xvzf .tar.gz` on the tarball to extract it. + 3. `cd` into the directory. + 4. Run `python setup.py install` to install the package and any dependencies. + ## Getting Started -Executing the script without any arguments shows the different arguments that can be used: + +Executing the script with the `-h` flag shows the different arguments that can be used: ```bash -$ sudo python3 iobs.py -iobs.py 0.3.1 -Usage: iobs.py [-c] [-l] [-o ] [-r ] [-v] [-x] -Command Line Arguments: - : The configuration file to use. --c : (OPTIONAL) The application will continue in the case of a job failure. --l : (OPTIONAL) Logs debugging information to an iobs.log file. --o : (OPTIONAL) Outputs metric information to a file. --r : (OPTIONAL) Used to retry a job more than once if failure occurs. Defaults to 1. --v : (OPTIONAL) Prints verbose information to the STDOUT. --x : (OPTIONAL) Attempts to clean up intermediate files. +$ iobs -h +usage: iobs [-h] [--version] {execute} + +positional arguments: + {execute} + +optional arguments: + -h, --help show this help message and exit + --version show program's version number and exit ``` -### Input -The main input to the tool should be a file in [INI](https://en.wikipedia.org/wiki/INI_file) format which has the configurations for how to run each of the workloads. The possible configuration settings for each workload is the following: -* command [str]- The workload generation command to run (e.x. fio ...). -* delay [int] - The amount of delay between running the workload and starting the trace. Defaults to 0. -* device [str] - The device to run the trace on (e.x. /dev/sdd). -* schedulers [str] - The io schedulers to use (e.x. noop, cfq, deadline, none, mq-deadline, bfq, kyber). -* repetition [int] - The number of times to repeat the workloads (will aggregate and average metrics). Defaults to 0. -* runtime [int] - The amount of time in seconds to run the trace (should match the workload runtime). -* workload [str] - The name of the workload generation tool (e.x. fio). - -Each of these commands should be under a header [...] indicating a specific job. The global header [global] can be used for configuring settings that are the same for all jobs. - -Example input files can be found under the [examples](https://github.com/UOFL-CSL/iobs/tree/master/examples) folder. - -### Output -Upon completion of each job, the following metrics are output to STDOUT: -* Latency [µs] -* Submission Latency [µs] -* Completion Latency [µs] -* File System Latency [µs] -* Block Layer Latency [µs] -* Device Latency [µs] -* IOPS -* Throughput [1024 MB/s] -* Total IO [KB] - -The following is a sample output the is given for a job: +### `iobs execute` + +Executes one or more `iobs` configuration files. + ```bash -(kyber) (/dev/nvme0n1): -Latency [µs]: (read): 5472.80 (write): 0.00 -Submission Latency [µs]: (read): 519.02 (write): 0.00 -Completion Latency [µs]: (read): 4952.97 (write): 0.00 -File System Latency [µs]: (read): 2.35 (write): -4950.62 -Block Layer Latency [µs]: 35.65 -Device Latency [µs]: 4914.97 -IOPS: (read) 182.51 (write) 0.00 -Throughput [1024 MB/s]: (read) 912.57 (write) 0.00 -Total IO [KB]: 560686080.00 +$ iobs execute -h +usage: iobs execute [-h] [-o OUTPUT_DIRECTORY] [-l LOG_FILE] + [--log-level {1,2,3}] [-s] [-r RETRY_COUNT] [-c] + input [input ...] + +positional arguments: + input The configuration files to execute. + +optional arguments: + -h, --help show this help message and exit + -o OUTPUT_DIRECTORY, --output-directory OUTPUT_DIRECTORY + The output directory for output files. Defaults to the + current working directory. + -l LOG_FILE, --log-file LOG_FILE + The file to log information to. + --log-level {1,2,3} The level of information to which to log: 1 (Debug), 2 + (Info), 3 (Error). Defaults to 2. + -s, --silent Silences output to STDOUT. + -r RETRY_COUNT, --retry-count RETRY_COUNT + Number of times to retry a failed workload. Defaults + to 1. + -c, --continue-on-failure + If a input fails, continues executing other inputs; + otherwise exits the program. +``` + +## Configuration Files + +Configuration files should be in [INI](https://en.wikipedia.org/wiki/INI_file) format. Each section should be contained +in square brackets [...] with configuration settings below the section. Different section names are reserved for special +purposes; all others are considered a workload section. Depending on the `workload_type` given under the `global` +section, additional settings may be permitted under a given section. + +Settings which accept lists of values should be separated by a comma `,`. + +### `global` + +Global settings for each of the workloads. + + * `workload_type` (required) - The type of workloads to run: `fio`. + * **ex:** `workload_type=fio` + * `devices` (required) - The devices to execute on. + * **ex:** `devices=/dev/nvme0n1/,/dev/nvme1n1` + * *NOTE: Should be valid block devices.* + * `schedulers` (required) - The schedulers to use on the device. + * **ex:** `schedulers=none,kyber,bfq,mq-deadline` + * *NOTE: Should be available for each device specified.* + * `repetitions` (optional) - The number of times to repeat the workloads. Defaults to 1. + * **ex:**: `repetitions=5` + +### `output` + +Output settings for what metrics to write in the output `.csv` files. The output file will have the same name +as the input configuration file and the extension replaced with `.csv`. Note that the last column in the output +file will always be `END`. + + * `format` (optional) - The metrics to write. Default and allowed format depend on `workload_type` (see below). + * `append_template` (optional) - Whether to append the `template` combinations. Defaults to True. + * **ex:** `append_template=1` + ** *NOTE: This should be set to `False` if a custom format is given which includes `template` information.* + +The `format` accepts a list of metric names which should be retrieved from the workload and written in the output files. +Each metric name can accept the full name or an abbreviated name (if there is one). Also, when using the `template` +section, the names of the `template` settings can be specified as well. For example, if the template setting +`rw=randread,randwrite` is specified, then the `rw` name can be used in the `format` to write the configuration used +in the workload. + +The following `format` metric names are used by any `workload_type`: + * `workload` (or `w`) - The name of the workload. + * `device` (or `d`) - The name of the device. + * `scheduler` (or `s`) - The name of the scheduler. + +**`fio`** + * `include_lat_percentiles` (optional) - Whether to include lat percentile metrics. Defaults to False. + * **ex:** `include_lat_percentiles=1` + * *NOTE: The `fio` workload file should have `lat_percentiles=1` set.* + * `include_clat_percentiles` (optional) - Whether to include clat percentile metrics. Defaults to False. + * **ex:** `include_clat_percentiles=1` + * *NOTE: The `fio` workload file should have `clat_percentiles=1` set (it is typically enabled by default).* + +The following `format` metric names are used by the `fio` `workload_type`: + * `job-runtime` (or `run`) - The total runtime for the job in ms. + * `total-ios-read` (or `tir`) - The total number of IOs read. + * `total-ios-write` (or `tiw`) - The total number of IOs written. + * `io-kbytes-read` (or `ibr`) - The total KB read. + * `io-kbytes-write` (or `ibw`) - The total KB written. + * `bw-read` (or `bwr`) - The average read bandwidth (throughput) in KB/s. + * `bw-write` (or `bww`) - The average write bandwidth (throughput) in KB/s. + * `iops-read` (or `opr`) - The average read IO/s. + * `iops-write` (or `ipw`) - The average write IO/s. + * `lat-min-read` (or `lir`) - The minimum read latency in ns. + * `lat-min-write` (or `liw`) - The minimum write latency in ns. + * `lat-max-read` (or `lar`) - The maximum read latency in ns. + * `lat-max-write` (or `law`) - The maximum write latency in ns. + * `lat-mean-read` (or `lmr`) - The average read latency in ns. + * `lat-mean-write` (or `lmw`) - The average write latency in ns. + * `lat-stddev-read` (or `lsr`) - The standard deviation of the read latency in ns. + * `lat-stddev-write` (or `lsw`) - The standard deviation of the write latency in ns. + * `slat-min-read` (or `sir`) - The minimum read submission latency in ns. + * `slat-min-write` (or `siw`) - The minimum write submission latency in ns. + * `slat-max-read` (or `sar`) - The maximum read submission latency in ns. + * `slat-max-write` (or `saw`) - The maximum write submission latency in ns. + * `slat-mean-read` (or `smr`) - The average read submission latency in ns. + * `slat-mean-write` (or `smw`) - The average write submission latency in ns. + * `slat-stddev-read` (or `ssr`) - The standard deviation of the read submission latency in ns. + * `slat-stddev-write` (or `ssw`) - The standard deviation of the write submission latency in ns. + * `clat-min-read` (or `cir`) - The minimum read completion latency in ns. + * `clat-min-write` (or `ciw`) - The minimum write completion latency in ns. + * `clat-max-read` (or `car`) - The maximum read completion latency in ns. + * `clat-max-write` (or `caw`) - The maximum write completion latency in ns. + * `clat-mean-read` (or `cmr`) - The average read completion latency in ns. + * `clat-mean-write` (or `cmw`) - The average write completion latency in ns. + * `clat-stddev-read` (or `csr`) - The standard deviation of the read completion latency in ns. + * `clat-stddev-writ` (or `csw`) - The standard deviation of the write completion latency in ns. + * `clat-percentile-read` (or `cpr`) - The read completion latency percentiles in ns. + * *NOTE: If `include_clat_percentile=1` is not set, this is ignored. The number of columns depends + on the number of percentiles reported by `fio`.* + * `clat-percentile-write` (or `cpw`) - The write completion latency percentiles in ns. + * *NOTE: If `include_clat_percentile=1` is not set, this is ignored. The number of columns depends + on the number of percentiles reported by `fio`.* + * `lat-percentile-read` (or `lpr`) - The read latency percentiles in ns. + * *NOTE: If `include_clat_percentile=1` is not set, this is ignored. The number of columns depends + on the number of percentiles reported by `fio`.* + * `lat-percentile-write` (or `lpw`) - The write latency percentiles in ns. + * *NOTE: If `include_clat_percentile=1` is not set, this is ignored. The number of columns depends + on the number of percentiles reported by `fio`.* + +The default `format` used if none is given is the following: + * `workload` + * `device` + * `scheduler` + * `job-runtime` + * `total-ios-read` + * `total-ios-write` + * `io-kbytes-read` + * `io-kbytes-write` + * `bw-read` + * `bw-write` + * `iops-read` + * `iops-write` + * `lat-min-read` + * `lat-min-write` + * `lat-max-read` + * `lat-max-write` + * `lat-mean-read` + * `lat-mean-write` + * `lat-stddev-read` + * `lat-stddev-write` + * `slat-min-read` + * `slat-min-write` + * `slat-max-read` + * `slat-max-write` + * `slat-mean-read` + * `slat-mean-write` + * `slat-stddev-read` + * `slat-stddev-write` + * `clat-min-read` + * `clat-min-write` + * `clat-max-read` + * `clat-max-write` + * `clat-mean-read` + * `clat-mean-write` + * `clat-stddev-read` + * `clat-stddev-write` + * `clat-percentile-read` + * `clat-percentile-write` + +### `template` + +Template settings for interpolating different setting combinations into `workload` files. + +* `enabled` (optional) - Whether to enable templating. Defaults to False. + * **ex:** `enabled=1` + +All other settings added under this section are used to interpolate the `workload` files. When files are interpolated, +an interpolated copy is made with the name `__temp__` appended to them. To provide settings to interpolate within a +`workload` file, the following syntax should be used: `<%setting-name%>`. By default, the following will always be +interpolated if templating is enabled: `<%device%>` the device, `<%device_name%>` the name of the device (i.e. no /dev/), +`<%scheduler%>` the scheduler. + +The following is an example `template` section: + +```ini +[template] +enabled=1 +rw=randread,randwrite +iodepth=1,2,4,8,16 ``` -In addition to STDOUT, an output file can be given via the `[-o] < output>` command-line argument. This file will be written to as a csv with the first row being a header of the following columns: -* device -* io-depth -* workload -* scheduler -* slat-read -* slat-write -* clat-read -* clat-write -* lat-read -* lat-write -* q2c -* d2c -* fslat-read -* fslat-write -* bslat -* iops-read -* iops-write -* throughput-read -* throughput-write -* io-kbytes -* start-time -* stop-time +In this example, the combination of the settings is (rw=randread, iodepth=1), (rw=randread, iodepth=2), ... The following +will be interpolated in the file `<%rw%>` and `<%iodepth%>` and replaced with their combination values. + +### workloads + +All other sections are considered a distinct workload that will be ran for each `device`, `scheduler`, and +`template setting` combination for a given number of `repetitions`. The name of the section is considered the name of +the workload. + + * `file` (required) - The file to execute. + * **ex:** `file=my-job.fio` + +The file will be executed with the appropriate command given the `workload_type`. For example, `workload_type=fio` would +run `fio --output-format=json`. + +## Examples +Usage examples can be found under the [examples](https://github.com/UOFL-CSL/iobs/tree/master/examples) folder. ## License -Copyright (c) 2018 UofL Computer Systems Lab. See [LICENSE](https://github.com/UOFL-CSL/iobs/blob/master/LICENSE) for details. +Copyright (c) 2018 UofL Computer Systems Lab. See [LICENSE](https://github.com/UOFL-CSL/iobs/tree/master/LICENSE) for details. From 26c86b4a0fffa2ff6f4c9281ec10eefc9d92d8f1 Mon Sep 17 00:00:00 2001 From: Jared Gillespie Date: Tue, 26 Feb 2019 16:37:14 -0500 Subject: [PATCH 16/21] Removing .idea files --- .idea/iobs.iml | 11 ----- .idea/misc.xml | 4 -- .idea/modules.xml | 8 --- .idea/vcs.xml | 6 --- .idea/workspace.xml | 116 -------------------------------------------- 5 files changed, 145 deletions(-) delete mode 100644 .idea/iobs.iml delete mode 100644 .idea/misc.xml delete mode 100644 .idea/modules.xml delete mode 100644 .idea/vcs.xml delete mode 100644 .idea/workspace.xml diff --git a/.idea/iobs.iml b/.idea/iobs.iml deleted file mode 100644 index 6711606..0000000 --- a/.idea/iobs.iml +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - - - - - \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml deleted file mode 100644 index 65531ca..0000000 --- a/.idea/misc.xml +++ /dev/null @@ -1,4 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml deleted file mode 100644 index b2b3c23..0000000 --- a/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 94a25f7..0000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml deleted file mode 100644 index 6873ec2..0000000 --- a/.idea/workspace.xml +++ /dev/null @@ -1,116 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -