From f023a1c457c20479af9adfc7783558b3043251aa Mon Sep 17 00:00:00 2001 From: Shashank Reddy Boyapally Date: Wed, 17 Jul 2024 14:22:58 -0400 Subject: [PATCH] add collapse in junit as a table, rename metrics in readout (#50) * add collapse in junit as a table, rename metrics in readout Signed-off-by: Shashank Reddy Boyapally * fixed trivial error Signed-off-by: Shashank Reddy Boyapally * made output shorter and concise Signed-off-by: Shashank Reddy Boyapally * changed format to explicitly state changepoints Signed-off-by: Shashank Reddy Boyapally --------- Signed-off-by: Shashank Reddy Boyapally --- examples/readout-control-plane-cdv2.yaml | 14 +- orion.py | 1 + pkg/edivisive.py | 36 ++-- pkg/isolationForest.py | 2 +- pkg/utils.py | 242 ++++++++++++++--------- 5 files changed, 184 insertions(+), 111 deletions(-) diff --git a/examples/readout-control-plane-cdv2.yaml b/examples/readout-control-plane-cdv2.yaml index bb256be..1fd0614 100644 --- a/examples/readout-control-plane-cdv2.yaml +++ b/examples/readout-control-plane-cdv2.yaml @@ -134,25 +134,25 @@ tests : controlPlaneArch: amd64 metrics : - - name: CPU_Usage + - name: CPU_Usage_kube-apiserver metricName.keyword: cpu-kube-apiserver metric_of_interest: value agg: value: kube_apiserver agg_type: avg - - name: Max_Aggregated_RSS_Usage + - name: Max_Aggregated_RSS_Usage_kube-apiserver metricName.keyword: max-memory-sum-kube-apiserver metric_of_interest: value agg: value: kube_apiserver agg_type: avg - - name: CPU_Usage + - name: CPU_Usage_etcd metricName.keyword: cpu-etcd metric_of_interest: value agg: value: etcd agg_type: avg - - name: Max_Aggregated_RSS_Usage + - name: Max_Aggregated_RSS_Usage_etcd metricName.keyword: max-memory-etcd metric_of_interest: value agg: @@ -227,21 +227,21 @@ tests : controlPlaneArch: amd64 metrics : - - name: Read_Only_API_request_P99_latency + - name: Read_Only_API_request_P99_latency_namespace metricName.keyword: avg-ro-apicalls-latency labels.scope.keyword: namespace metric_of_interest: value agg: value: namespace_scoped agg_type: avg - - name: Read_Only_API_request_P99_latency + - name: Read_Only_API_request_P99_latency_cluster metricName.keyword: avg-ro-apicalls-latency labels.scope.keyword: cluster metric_of_interest: value agg: value: cluster_scoped agg_type: avg - - name: Read_Only_API_request_P99_latency + - name: Read_Only_API_request_P99_latency_avg metricName.keyword: avg-mutating-apicalls-latency metric_of_interest: value agg: diff --git a/orion.py b/orion.py index ae95088..68d0c5a 100644 --- a/orion.py +++ b/orion.py @@ -103,6 +103,7 @@ def cli(max_content_width=120): # pylint: disable=unused-argument ) @click.option("--lookback", help="Get data from last X days and Y hours. Format in XdYh") @click.option("--convert-tinyurl", is_flag=True, help="Convert buildUrls to tiny url format for better formatting") +@click.option("--collapse", is_flag=True, help="Only outputs changepoints, previous and later runs in the xml format") def cmd_analysis(**kwargs): """ Orion runs on command line mode, and helps in detecting regressions diff --git a/pkg/edivisive.py b/pkg/edivisive.py index 6279d70..7194a7c 100644 --- a/pkg/edivisive.py +++ b/pkg/edivisive.py @@ -1,5 +1,6 @@ """EDivisive Algorithm from hunter""" -#pylint: disable = line-too-long + +# pylint: disable = line-too-long import json import pandas as pd from hunter.report import Report, ReportType @@ -8,7 +9,6 @@ from pkg.utils import json_to_junit - class EDivisive(Algorithm): """Implementation of the EDivisive algorithm using hunter @@ -36,7 +36,10 @@ def output_json(self): (change_point.stats.mean_2 - change_point.stats.mean_1) / change_point.stats.mean_1 ) * 100 - if percentage_change * self.metrics_config[key]["direction"] > 0 or self.metrics_config[key]["direction"]==0: + if ( + percentage_change * self.metrics_config[key]["direction"] > 0 + or self.metrics_config[key]["direction"] == 0 + ): dataframe_json[index]["metrics"][key][ "percentage_change" ] = percentage_change @@ -53,14 +56,19 @@ def output_text(self): def output_junit(self): test_name, data_json = self.output_json() - data_json=json.loads(data_json) - data_junit = json_to_junit(test_name=test_name, data_json=data_json, metrics_config=self.metrics_config) + data_json = json.loads(data_json) + data_junit = json_to_junit( + test_name=test_name, data_json=data_json, metrics_config=self.metrics_config, options=self.options + ) return test_name, data_junit def _analyze(self): self.dataframe["timestamp"] = pd.to_datetime(self.dataframe["timestamp"]) self.dataframe["timestamp"] = self.dataframe["timestamp"].astype(int) // 10**9 - metrics = {column: Metric(value.get("direction",1), 1.0) for column,value in self.metrics_config.items()} + metrics = { + column: Metric(value.get("direction", 1), 1.0) + for column, value in self.metrics_config.items() + } data = {column: self.dataframe[column] for column in self.metrics_config} attributes = { column: self.dataframe[column] @@ -79,17 +87,23 @@ def _analyze(self): # filter by direction for change_point_group in change_points: change_point_group.changes = [ - change for change in change_point_group.changes + change + for change in change_point_group.changes if not ( - (self.metrics_config[change.metric]["direction"] == 1 and change.stats.mean_1 > change.stats.mean_2) or - (self.metrics_config[change.metric]["direction"] == -1 and change.stats.mean_1 < change.stats.mean_2) + ( + self.metrics_config[change.metric]["direction"] == 1 + and change.stats.mean_1 > change.stats.mean_2 + ) + or ( + self.metrics_config[change.metric]["direction"] == -1 + and change.stats.mean_1 < change.stats.mean_2 + ) ) ] - for i in range(len(change_points)-1,-1,-1): + for i in range(len(change_points) - 1, -1, -1): if len(change_points[i].changes) == 0: del change_points[i] - report = Report(series, change_points) return report, series diff --git a/pkg/isolationForest.py b/pkg/isolationForest.py index 0812430..3edd620 100644 --- a/pkg/isolationForest.py +++ b/pkg/isolationForest.py @@ -54,7 +54,7 @@ def output_text(self): def output_junit(self): test_name, data_json = self.output_json() data_json=json.loads(data_json) - data_junit = json_to_junit(test_name=test_name, data_json=data_json, metrics_config=self.metrics_config) + data_junit = json_to_junit(test_name=test_name, data_json=data_json, metrics_config=self.metrics_config, options=self.options) return test_name, data_junit def analyze(self, dataframe: pd.DataFrame): diff --git a/pkg/utils.py b/pkg/utils.py index cae427a..b09339d 100644 --- a/pkg/utils.py +++ b/pkg/utils.py @@ -1,5 +1,5 @@ # pylint: disable=cyclic-import -# pylint: disable = line-too-long, too-many-arguments +# pylint: disable = line-too-long, too-many-arguments, consider-using-enumerate """ module for all utility functions orion uses """ @@ -12,6 +12,7 @@ import xml.etree.ElementTree as ET import xml.dom.minidom from datetime import datetime, timedelta, timezone +from tabulate import tabulate import yaml import pandas as pd @@ -20,6 +21,7 @@ from fmatch.logrus import SingletonLogger + # pylint: disable=too-many-locals def get_metric_data(ids, index, metrics, match, metrics_config): """Gets details metrics basked on metric yaml list @@ -34,11 +36,11 @@ def get_metric_data(ids, index, metrics, match, metrics_config): Returns: dataframe_list: dataframe of the all metrics """ - logger_instance= SingletonLogger.getLogger("Orion") + logger_instance = SingletonLogger.getLogger("Orion") dataframe_list = [] for metric in metrics: - labels=metric.pop("labels",None) - direction = int(metric.pop("direction",0)) + labels = metric.pop("labels", None) + direction = int(metric.pop("direction", 0)) metric_name = metric["name"] logger_instance.info("Collecting %s", metric_name) metric_of_interest = metric["metric_of_interest"] @@ -49,13 +51,15 @@ def get_metric_data(ids, index, metrics, match, metrics_config): agg_value = metric["agg"]["value"] agg_type = metric["agg"]["agg_type"] agg_name = agg_value + "_" + agg_type - cpu_df = match.convert_to_df(cpu, columns=["uuid", "timestamp", agg_name]) - cpu_df= cpu_df.drop_duplicates(subset=['uuid'],keep='first') - metric_dataframe_name= f"{metric_name}_{agg_type}" + cpu_df = match.convert_to_df( + cpu, columns=["uuid", "timestamp", agg_name] + ) + cpu_df = cpu_df.drop_duplicates(subset=["uuid"], keep="first") + metric_dataframe_name = f"{metric_name}_{agg_type}" cpu_df = cpu_df.rename(columns={agg_name: metric_dataframe_name}) - metric["labels"]=labels - metric["direction"]=direction - metrics_config[metric_dataframe_name]=metric + metric["labels"] = labels + metric["direction"] = direction + metrics_config[metric_dataframe_name] = metric dataframe_list.append(cpu_df) logger_instance.debug(cpu_df) @@ -71,13 +75,14 @@ def get_metric_data(ids, index, metrics, match, metrics_config): podl_df = match.convert_to_df( podl, columns=["uuid", "timestamp", metric_of_interest] ) - metric_dataframe_name=f"{metric_name}_{metric_of_interest}" + metric_dataframe_name = f"{metric_name}_{metric_of_interest}" podl_df = podl_df.rename( - columns={metric_of_interest: metric_dataframe_name}) - metric["labels"]=labels - metric["direction"]=direction - metrics_config[metric_dataframe_name]=metric - podl_df=podl_df.drop_duplicates() + columns={metric_of_interest: metric_dataframe_name} + ) + metric["labels"] = labels + metric["direction"] = direction + metrics_config[metric_dataframe_name] = metric + podl_df = podl_df.drop_duplicates() dataframe_list.append(podl_df) logger_instance.debug(podl_df) except Exception as e: # pylint: disable=broad-exception-caught @@ -98,7 +103,7 @@ def get_metadata(test): Returns: dict: dictionary of the metadata """ - logger_instance= SingletonLogger.getLogger("Orion") + logger_instance = SingletonLogger.getLogger("Orion") metadata = test["metadata"] metadata["ocpVersion"] = str(metadata["ocpVersion"]) logger_instance.debug("metadata" + str(metadata)) @@ -115,7 +120,7 @@ def load_config(config): Returns: dict: dictionary of the config file """ - logger_instance= SingletonLogger.getLogger("Orion") + logger_instance = SingletonLogger.getLogger("Orion") try: with open(config, "r", encoding="utf-8") as file: data = yaml.safe_load(file) @@ -139,7 +144,7 @@ def get_es_url(data): Returns: str: es url """ - logger_instance= SingletonLogger.getLogger("Orion") + logger_instance = SingletonLogger.getLogger("Orion") if "ES_SERVER" in data.keys(): return data["ES_SERVER"] if "ES_SERVER" in os.environ: @@ -159,34 +164,44 @@ def get_ids_from_index(metadata, fingerprint_index, uuids, match, baseline): Returns: _type_: index and uuids """ - if metadata["benchmark.keyword"] in ["ingress-perf","k8s-netperf"] : + if metadata["benchmark.keyword"] in ["ingress-perf", "k8s-netperf"]: return uuids if baseline == "": - runs = match.match_kube_burner(uuids,fingerprint_index) + runs = match.match_kube_burner(uuids, fingerprint_index) ids = match.filter_runs(runs, runs) else: ids = uuids return ids -def get_build_urls(index, uuids,match): - """Gets metadata of the run from each test + +def get_build_urls(index, uuids, match): + """Gets metadata of the run from each test to get the build url Args: uuids (list): str list of uuid to find build urls of match: the fmatch instance - + Returns: dict: dictionary of the metadata """ - test = match.getResults("",uuids,index,{}) + test = match.getResults("", uuids, index, {}) buildUrls = {run["uuid"]: run["buildUrl"] for run in test} return buildUrls -def process_test(test, match, output, uuid, baseline, metrics_config, start_timestamp, convert_tinyurl): +def process_test( + test, + match, + output, + uuid, + baseline, + metrics_config, + start_timestamp, + convert_tinyurl, +): """generate the dataframe for the test given Args: @@ -198,34 +213,36 @@ def process_test(test, match, output, uuid, baseline, metrics_config, start_time Returns: _type_: merged dataframe """ - logger_instance= SingletonLogger.getLogger("Orion") - benchmarkIndex=test['benchmarkIndex'] - fingerprint_index=test['index'] - if uuid in ('', None): + logger_instance = SingletonLogger.getLogger("Orion") + benchmarkIndex = test["benchmarkIndex"] + fingerprint_index = test["index"] + if uuid in ("", None): metadata = get_metadata(test) else: - metadata = filter_metadata(uuid,match) + metadata = filter_metadata(uuid, match) logger_instance.info("The test %s has started", test["name"]) runs = match.get_uuid_by_metadata(metadata, lookback_date=start_timestamp) uuids = [run["uuid"] for run in runs] buildUrls = {run["uuid"]: run["buildUrl"] for run in runs} - if baseline in ('', None): + if baseline in ("", None): if len(uuids) == 0: logger_instance.error("No UUID present for given metadata") return None else: - uuids = [uuid for uuid in re.split(' |,',baseline) if uuid] + uuids = [uuid for uuid in re.split(" |,", baseline) if uuid] uuids.append(uuid) - buildUrls = get_build_urls(fingerprint_index, uuids,match) - fingerprint_index=benchmarkIndex + buildUrls = get_build_urls(fingerprint_index, uuids, match) + fingerprint_index = benchmarkIndex ids = get_ids_from_index(metadata, fingerprint_index, uuids, match, baseline) metrics = test["metrics"] - dataframe_list = get_metric_data(ids, fingerprint_index, metrics, match, metrics_config) + dataframe_list = get_metric_data( + ids, fingerprint_index, metrics, match, metrics_config + ) for i, df in enumerate(dataframe_list): - if i != 0 and ('timestamp' in df.columns): - dataframe_list[i] = df.drop(columns=['timestamp']) + if i != 0 and ("timestamp" in df.columns): + dataframe_list[i] = df.drop(columns=["timestamp"]) merged_df = reduce( lambda left, right: pd.merge(left, right, on="uuid", how="inner"), @@ -233,58 +250,63 @@ def process_test(test, match, output, uuid, baseline, metrics_config, start_time ) shortener = pyshorteners.Shortener(timeout=10) merged_df["buildUrl"] = merged_df["uuid"].apply( - lambda uuid: shortener.tinyurl.short(buildUrls[uuid]) - if convert_tinyurl else buildUrls[uuid] #pylint: disable = cell-var-from-loop - ) + lambda uuid: ( + shortener.tinyurl.short(buildUrls[uuid]) + if convert_tinyurl + else buildUrls[uuid] + ) # pylint: disable = cell-var-from-loop + ) output_file_path = output.split(".")[0] + "-" + test["name"] + ".csv" match.save_results(merged_df, csv_file_path=output_file_path) return merged_df -def filter_metadata(uuid,match): + +def filter_metadata(uuid, match): """Gets metadata of the run from each test Args: uuid (str): str of uuid ot find metadata of match: the fmatch instance - + Returns: dict: dictionary of the metadata """ - logger_instance= SingletonLogger.getLogger("Orion") + logger_instance = SingletonLogger.getLogger("Orion") test = match.get_metadata_by_uuid(uuid) metadata = { - 'platform': '', - 'clusterType': '', - 'masterNodesCount': 0, - 'workerNodesCount': 0, - 'infraNodesCount': 0, - 'masterNodesType': '', - 'workerNodesType': '', - 'infraNodesType': '', - 'totalNodesCount': 0, - 'ocpVersion': '', - 'networkType': '', - 'ipsec': '', - 'fips': '', - 'encrypted': '', - 'publish': '', - 'computeArch': '', - 'controlPlaneArch': '' + "platform": "", + "clusterType": "", + "masterNodesCount": 0, + "workerNodesCount": 0, + "infraNodesCount": 0, + "masterNodesType": "", + "workerNodesType": "", + "infraNodesType": "", + "totalNodesCount": 0, + "ocpVersion": "", + "networkType": "", + "ipsec": "", + "fips": "", + "encrypted": "", + "publish": "", + "computeArch": "", + "controlPlaneArch": "", } - for k,v in test.items(): + for k, v in test.items(): if k not in metadata: continue metadata[k] = v - metadata['benchmark.keyword'] = test['benchmark'] + metadata["benchmark.keyword"] = test["benchmark"] metadata["ocpVersion"] = str(metadata["ocpVersion"]) - #Remove any keys that have blank values + # Remove any keys that have blank values no_blank_meta = {k: v for k, v in metadata.items() if v} - logger_instance.debug('No blank metadata dict: ' + str(no_blank_meta)) + logger_instance.debug("No blank metadata dict: " + str(no_blank_meta)) return no_blank_meta -def json_to_junit(test_name, data_json, metrics_config): + +def json_to_junit(test_name, data_json, metrics_config, options): """Convert json to junit format Args: @@ -299,30 +321,17 @@ def json_to_junit(test_name, data_json, metrics_config): testsuites, "testsuite", name=f"{test_name} nightly compare" ) failures_count = 0 - test_count=0 - for run in data_json: - run_data = { - str(key): str(value).lower() - for key, value in run.items() - if key in ["timestamp"] - } - for metric, value in run["metrics"].items(): - test_count+=1 - failure = "false" - if not value["percentage_change"] == 0: - failure = "true" - failures_count += 1 - labels = metrics_config[metric]["labels"] - label_string = " ".join(labels) if labels else "" - testcase = ET.SubElement( - testsuite, - "testcase", - name=f"{label_string} {metric} regression detection", - attrib=run_data, - ) - if failure == "true": - failure_element = ET.SubElement(testcase, "failure") - failure_element.text = f"{metric} has a value of {value['value']:.2f} with a percentage change of {value['percentage_change']:.2f}% over the previous runs" + test_count = 0 + for metric, value in metrics_config.items(): + test_count += 1 + labels = value["labels"] + label_string = " ".join(labels) if labels else "" + testcase = ET.SubElement(testsuite, "testcase", name=f"{label_string} {metric} regression detection", timestamp=str(int(datetime.now().timestamp()))) + if [run for run in data_json if not run["metrics"][metric]["percentage_change"] == 0]: + failures_count +=1 + failure = ET.SubElement(testcase,"failure") + failure.text = "\n"+generate_tabular_output(data_json, metric_name=metric, collapse=options["collapse"])+"\n" + testsuite.set("failures", str(failures_count)) testsuite.set("tests", str(test_count)) xml_str = ET.tostring(testsuites, encoding="utf8", method="xml").decode() @@ -330,6 +339,55 @@ def json_to_junit(test_name, data_json, metrics_config): pretty_xml_as_string = dom.toprettyxml() return pretty_xml_as_string +def generate_tabular_output(data: list, metric_name: str, collapse: bool) -> str: + """converts json to tabular format + + Args: + data (list):data in json format + metric_name (str): metric name + Returns: + str: tabular form of data + """ + records = [] + create_record = lambda record: { # pylint: disable = C3001 + "uuid": record["uuid"], + "timestamp": datetime.fromtimestamp(record["timestamp"], timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ'), + "buildUrl": record["buildUrl"], + metric_name: record["metrics"][metric_name]["value"], + "is_changepoint": record["is_changepoint"], + "percentage_change": record["metrics"][metric_name]["percentage_change"], + } + if collapse: + for i in range(1, len(data)): + if data[i]["metrics"][metric_name]["percentage_change"] != 0: + records.append(create_record(data[i-1])) + records.append(create_record(data[i])) + if i + 1 < len(data): + records.append(create_record(data[i+1])) + else: + for i in range(0,len(data)): + records.append(create_record(data[i])) + + df = pd.DataFrame(records).drop_duplicates().reset_index(drop=True) + table = tabulate(df, headers='keys', tablefmt='psql') + lines = table.split('\n') + highlighted_lines = [] + if lines: + highlighted_lines+=lines[0:3] + for i, line in enumerate(lines[3:-1]): + if df['is_changepoint'][i]: # Offset by 3 to account for header and separator + highlighted_line = f"{lines[i+3]} -- changepoint" + highlighted_lines.append(highlighted_line) + else: + highlighted_lines.append(line) + highlighted_lines.append(lines[-1]) + +# Join the lines back into a single string + highlighted_table = '\n'.join(highlighted_lines) + + return highlighted_table + + def get_subtracted_timestamp(time_duration: str) -> datetime: """Get subtracted datetime from now @@ -339,8 +397,8 @@ def get_subtracted_timestamp(time_duration: str) -> datetime: Returns: datetime: return datetime of given timegap from now """ - logger_instance= SingletonLogger.getLogger("Orion") - reg_ex = re.match(r'^(?:(\d+)d)?(?:(\d+)h)?$', time_duration) + logger_instance = SingletonLogger.getLogger("Orion") + reg_ex = re.match(r"^(?:(\d+)d)?(?:(\d+)h)?$", time_duration) if not reg_ex: logger_instance.error("Wrong format for time duration, please provide in XdYh") days = int(reg_ex.group(1)) if reg_ex.group(1) else 0