tools: add scripts for automating tests of old tool revisions

Signed-off-by: Piotr Binkowski <pbinkowski@antmicro.com>

tools/compare.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+rm -r ./tests/generated
+make generate-tests
+
+for i in $(seq 0 $1); do
+	DATE=$(date --date="`date` -$i week" '+%Y-%m-%d')
+
+	rm -r ./out/
+	rm -rf ./third_party/tools
+	git submodule update
+
+	echo $DATE
+
+	for d in ./third_party/tools/*/ ; do
+		pushd $d
+		git checkout `git rev-list -n 1 --first-parent --before="$DATE" HEAD`
+		popd
+	done
+
+	make runners -j`nproc`
+
+	make -j`nproc`
+
+	cp ./out/report/report.csv $DATE.csv
+done
+
+./tools/csv-analyzer *.csv

tools/csv-analyzer

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+
+import argparse
+import csv
+import os
+
+parser = argparse.ArgumentParser()
+
+parser.add_argument("files", metavar='N', type=str, nargs='+')
+
+args = parser.parse_args()
+
+results = {}
+
+for f in args.files:
+    name = os.path.splitext(os.path.basename(f))[0]
+    results[name] = {}
+    with open(f, newline='') as csvfile:
+        reader = csv.DictReader(csvfile)
+        runners = reader.fieldnames[3:]
+        for runner in runners:
+            results[name][runner] = 0
+        for row in reader:
+            for runner in runners:
+                if row[runner] == 'True':
+                    results[name][runner] += 1
+
+with open('out.csv', 'w', newline='') as csvfile:
+    writer = csv.writer(csvfile)
+    writer.writerow(['date', *runners])
+    for d in results:
+        row = [d]
+        for runner in runners:
+            row.append(results[d][runner])
+        writer.writerow(row)