Skip to content

Commit 765ec53

Browse files
shashank-boyapallyshashank-boyapally
authored
orion cli, pylint workflow and README (#2)
1 parent dea10e6 commit 765ec53

File tree

6 files changed

+295
-0
lines changed

6 files changed

+295
-0
lines changed

‎.github/workflows/pylint.yml

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
name: Pylint
2+
3+
on: [push,pull_request]
4+
5+
jobs:
6+
build:
7+
runs-on: ubuntu-latest
8+
strategy:
9+
matrix:
10+
python-version: ["3.8", "3.9", "3.10", "3.11"]
11+
steps:
12+
- uses: actions/checkout@v3
13+
- name: Set up Python ${{ matrix.python-version }}
14+
uses: actions/setup-python@v3
15+
with:
16+
python-version: ${{ matrix.python-version }}
17+
- name: Install dependencies
18+
run: |
19+
python -m pip install --upgrade pip
20+
pip install pylint
21+
- name: Analysing the code with pylint
22+
run: |
23+
pylint -d C0103 $(git ls-files '*.py')

‎.gitignore

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -158,3 +158,7 @@ cython_debug/
158158
# and can be added to the global gitignore or merged into this file. For a more nuclear
159159
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
160160
#.idea/
161+
162+
*.yaml
163+
*.csv
164+
.vscode/

‎README.md

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
# Orion - CLI tool to find regressions
2+
Orion stands as a powerful command-line tool designed for identifying regressions within perf-scale CPT runs, leveraging metadata provided during the process. The detection mechanism relies on [hunter](https://github.com/datastax-labs/hunter).
3+
4+
Below is an illustrative example of the config and metadata that Orion can handle:
5+
6+
```
7+
tests :
8+
- name : aws-small-scale-cluster-density-v2
9+
platform: AWS
10+
masterNodesType: m6a.xlarge
11+
masterNodesCount: 3
12+
workerNodesType: m6a.xlarge
13+
workerNodesCount: 24
14+
benchmark: cluster-density-v2
15+
ocpVersion: 4.15
16+
networkType: OVNKubernetes
17+
# encrypted: true
18+
# fips: false
19+
# ipsec: false
20+
21+
metrics :
22+
- metric : podReadyLatency
23+
metricType : latency
24+
25+
- metric : apiserverCPU
26+
metricType : cpu
27+
namespace: openshift-kube-apiserver
28+
29+
- metric: ovnCPU
30+
metricType: cpu
31+
namespace: openshift-ovn-kubernetes
32+
33+
- metric: etcdCPU
34+
metricType: cpu
35+
namespace: openshift-ovn-kubernetes
36+
37+
38+
```
39+
40+
## Build Orion
41+
Building Orion is a straightforward process. Follow these commands:
42+
43+
Clone the current repository using git clone.
44+
45+
```
46+
>> git clone <repository_url>
47+
>> pip install venv
48+
>> source venv/bin/activate
49+
>> pip install -r requirements.txt
50+
>> export ES_SERVER = <es_server_url>
51+
>> pip install .
52+
```
53+
## Run Orion
54+
Executing Orion is as simple as building it. After following the build steps, run the following:
55+
```
56+
>> orion
57+
```
58+
At the moment,
59+
60+
Orion provides flexibility in configuring its behavior by allowing users to set the path to their config file using the ```--config``` flag.
61+
62+
For enhanced troubleshooting and debugging, Orion supports the ```--debug``` flag, enabling the generation of detailed debug logs.
63+
64+
Additionally, users can specify a custom path for the output CSV file using the ```--output``` flag, providing control over the location where the generated CSV will be stored.
65+
66+
Orion's seamless integration with metadata and hunter ensures a robust regression detection tool for perf-scale CPT runs.
67+
68+

‎orion.py

Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
"""
2+
This is the cli file for orion, tool to detect regressions using hunter
3+
"""
4+
# pylint: disable = import-error
5+
import sys
6+
from functools import reduce
7+
import logging
8+
import os
9+
10+
import click
11+
import yaml
12+
import pandas as pd
13+
from fmatch.matcher import Matcher
14+
15+
16+
@click.group()
17+
def cli():
18+
"""
19+
cli function to group commands
20+
"""
21+
22+
23+
# pylint: disable=too-many-locals
24+
@click.command()
25+
@click.option("--config", default="config.yaml", help="Path to the configuration file")
26+
@click.option("--output", default="output.csv", help="Path to save the output csv file")
27+
@click.option("--debug", is_flag=True, help="log level ")
28+
def orion(config, debug, output):
29+
"""Orion is the cli tool to detect regressions over the runs
30+
31+
Args:
32+
config (str): path to the config file
33+
debug (bool): lets you log debug mode
34+
output (str): path to the output csv file
35+
"""
36+
level = logging.DEBUG if debug else logging.INFO
37+
logger = logging.getLogger("Orion")
38+
logger = set_logging(level, logger)
39+
40+
if "ES_SERVER" not in os.environ:
41+
logger.error("ES_SERVER environment variable not set")
42+
sys.exit(1)
43+
44+
try:
45+
with open(config, "r", encoding="utf-8") as file:
46+
data = yaml.safe_load(file)
47+
logger.debug("The %s file has successfully loaded", config)
48+
except FileNotFoundError as e:
49+
logger.error("Config file not found: %s", e)
50+
sys.exit(1)
51+
except Exception as e: # pylint: disable=broad-exception-caught
52+
logger.error("An error occurred: %s", e)
53+
sys.exit(1)
54+
for test in data["tests"]:
55+
metadata = get_metadata(test)
56+
logger.info("The test %s has started", test["name"])
57+
match = Matcher(index="perf_scale_ci", level=level)
58+
uuids = match.get_uuid_by_metadata(metadata)
59+
if len(uuids) == 0:
60+
print("No UUID present for given metadata")
61+
sys.exit()
62+
63+
runs = match.match_kube_burner(uuids)
64+
ids = match.filter_runs(runs, runs)
65+
metrics = test["metrics"]
66+
dataframe_list = []
67+
68+
for metric in metrics:
69+
logger.info("Collecting %s", metric["metric"])
70+
if metric["metricType"] == "latency":
71+
if metric["metric"] == "podReadyLatency":
72+
try:
73+
podl = match.burner_results("", ids, "ripsaw-kube-burner*")
74+
podl_df = match.convert_to_df(
75+
podl, columns=["uuid", "timestamp", "P99"]
76+
)
77+
dataframe_list.append(podl_df)
78+
logger.debug(podl_df)
79+
except Exception as e: # pylint: disable=broad-exception-caught
80+
logger.error(
81+
"The namespace %s does not exist, exception %s",
82+
metric["namespace"],
83+
e,
84+
)
85+
86+
elif metric["metricType"] == "cpu":
87+
try:
88+
cpu = match.burner_cpu_results(
89+
ids, metric["namespace"], "ripsaw-kube-burner*"
90+
)
91+
cpu_df = match.convert_to_df(cpu, columns=["uuid", "cpu_avg"])
92+
cpu_df = cpu_df.rename(
93+
columns={"cpu_avg": metric["metric"] + "_cpu_avg"}
94+
)
95+
dataframe_list.append(cpu_df)
96+
logger.debug(cpu_df)
97+
except Exception as e: # pylint: disable=broad-exception-caught
98+
logger.error(
99+
"The namespace %s does not exist, exception %s",
100+
metric["namespace"],
101+
e,
102+
)
103+
104+
merged_df = reduce(
105+
lambda left, right: pd.merge(left, right, on="uuid", how="inner"),
106+
dataframe_list,
107+
)
108+
match.save_results(merged_df, csv_file_path=output)
109+
110+
111+
def get_metadata(test):
112+
"""Gets metadata of the run from each test
113+
114+
Args:
115+
test (dict): test dictionary
116+
117+
Returns:
118+
dict: dictionary of the metadata
119+
"""
120+
metadata_columns = [
121+
"platform",
122+
"masterNodesType",
123+
"masterNodesCount",
124+
"workerNodesType",
125+
"workerNodesCount",
126+
"benchmark",
127+
"ocpVersion",
128+
"networkType",
129+
"encrypted",
130+
"fips",
131+
"ipsec",
132+
]
133+
metadata = {key: test[key] for key in metadata_columns if key in test}
134+
metadata["ocpVersion"] = str(metadata["ocpVersion"])
135+
return metadata
136+
137+
138+
def set_logging(level, logger):
139+
"""sets log level and format
140+
141+
Args:
142+
level (_type_): level of the log
143+
logger (_type_): logger object
144+
145+
Returns:
146+
logging.Logger: a formatted and level set logger
147+
"""
148+
logger.setLevel(level)
149+
handler = logging.StreamHandler(sys.stdout)
150+
handler.setLevel(level)
151+
formatter = logging.Formatter(
152+
"%(asctime)s - %(name)s - %(levelname)s - %(message)s"
153+
)
154+
handler.setFormatter(formatter)
155+
logger.addHandler(handler)
156+
return logger
157+
158+
159+
if __name__ == "__main__":
160+
cli.add_command(orion)
161+
cli()

‎requirements.txt

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
certifi==2023.11.17
2+
click==8.1.7
3+
elastic-transport==8.11.0
4+
elasticsearch==8.11.1
5+
elasticsearch7==7.13.0
6+
fmatch==0.0.2
7+
numpy==1.26.3
8+
pandas==2.1.4
9+
python-dateutil==2.8.2
10+
pytz==2023.3.post1
11+
PyYAML==6.0.1
12+
six==1.16.0
13+
tzdata==2023.4
14+
urllib3==1.26.18

‎setup.py

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
# orion/setup.py
2+
"""
3+
setup.py for orion cli tool
4+
"""
5+
from setuptools import setup
6+
7+
setup(
8+
name='orion',
9+
version='1.0',
10+
py_modules=['orion'],
11+
install_requires=[
12+
'click',
13+
'fmatch'
14+
],
15+
entry_points={
16+
'console_scripts': [
17+
'orion = orion:orion',
18+
],
19+
},
20+
classifiers=[
21+
'Programming Language :: Python :: 3',
22+
'License :: OSI Approved :: MIT License',
23+
'Operating System :: OS Independent',
24+
],
25+
)

0 commit comments

Comments
 (0)