Benchmark run as module

benchmarks/README.md → benchmark/README.md

@@ -24,13 +24,13 @@ rm AutoShot.tar.gz
 ```
 
 ## Evaluation
-To evaluate PySceneDetect on a dataset, run the following command:
+To evaluate PySceneDetect on a dataset, run the following command from the root of the repo:
 ```
-python benchmark.py -d <dataset_name> --detector <detector_name>
+python -m benchmark -d <dataset_name> --detector <detector_name>
 ```
 For example, to evaluate ContentDetector on the BBC dataset:
 ```
-python evaluate.py -d BBC --detector detect-content
+python -m benchmark -d BBC --detector detect-content
 ```
 
 ### Result
@@ -40,10 +40,10 @@ The performance is computed as recall, precision, f1, and elapsed time.
 
 |      Detector     | Recall | Precision |   F1  | Elapsed time (second) |
 |:-----------------:|:------:|:---------:|:-----:|:---------------------:|
-|  AdaptiveDetector |  87.52 |   97.21   | 92.11 |         27.84         |
-|  ContentDetector  |  85.23 |   89.53   | 87.33 |         26.46         |
-|    HashDetector   |  92.96 |   76.27   | 83.79 |         16.26         |
-| HistogramDetector |  90.55 |   72.76   | 80.68 |         16.13         |
+|  AdaptiveDetector |  87.12 |   96.55   | 91.59 |         27.84         |
+|  ContentDetector  |  84.70 |   88.77   | 86.69 |         28.20         |
+|    HashDetector   |  92.30 |   75.56   | 83.10 |         16.00         |
+| HistogramDetector |  89.84 |   72.03   | 79.96 |         15.13         |
 | ThresholdDetector |  0.00  |   0.00    |  0.00 |         18.95         |
 
 #### AutoShot

benchmarks/benchmark.py → benchmark/__main__.py

@@ -1,12 +1,12 @@
 import argparse
 import time
+import os
 
-from bbc_dataset import BBCDataset
-from autoshot_dataset import AutoShotDataset
-
-from evaluator import Evaluator
 from tqdm import tqdm
 
+from benchmark.autoshot_dataset import AutoShotDataset
+from benchmark.bbc_dataset import BBCDataset
+from benchmark.evaluator import Evaluator
 from scenedetect import (
     AdaptiveDetector,
     ContentDetector,
@@ -18,22 +18,27 @@
 
 
 def _make_detector(detector_name: str):
-    detector_map = {
-        "detect-adaptive": AdaptiveDetector(),
-        "detect-content": ContentDetector(),
-        "detect-hash": HashDetector(),
-        "detect-hist": HistogramDetector(),
-        "detect-threshold": ThresholdDetector(),
-    }
-    return detector_map[detector_name]
+    if detector_name == "detect-adaptive":
+        return AdaptiveDetector()
+    if detector_name == "detect-content":
+        return ContentDetector()
+    if detector_name == "detect-hash":
+        return HashDetector()
+    if detector_name == "detect-hist":
+        return HistogramDetector()
+    if detector_name == "detect-threshold":
+        return ThresholdDetector()
+    raise RuntimeError(f"Unknown detector: {detector_name}")
+
 
+_DATASETS = {
+    "BBC": BBCDataset("benchmark/BBC"),
+    "AutoShot": AutoShotDataset("benchmark/AutoShot"),
+}
 
-def _make_dataset(dataset_name: str):
-    dataset_map = {
-        "BBC": BBCDataset("BBC"),
-        "AutoShot": AutoShotDataset("AutoShot"),
-    }
-    return dataset_map[dataset_name]
+_RESULT_PRINT_FORMAT = (
+    "Recall: {recall:.2f}, Precision: {precision:.2f}, F1: {f1:.2f} Elapsed time: {elapsed:.2f}\n"
+)
 
 
 def _detect_scenes(detector_type: str, dataset):
@@ -43,34 +48,28 @@ def _detect_scenes(detector_type: str, dataset):
         detector = _make_detector(detector_type)
         pred_scene_list = detect(video_file, detector)
         elapsed = time.time() - start
+        filename = os.path.basename(video_file)
         scenes = {
             scene_file: {
-                "video_file": video_file,
+                "video_file": filename,
                 "elapsed": elapsed,
                 "pred_scenes": [scene[1].frame_num for scene in pred_scene_list],
             }
         }
         result = Evaluator().evaluate_performance(scenes)
-        print(f"{video_file} results:")
-        print(
-            "Recall: {:.2f}, Precision: {:.2f}, F1: {:.2f} Elapsed time: {:.2f}\n".format(
-                result["recall"], result["precision"], result["f1"], result["elapsed"]
-            )
-        )
+        print(f"\n{filename} results:")
+        print(_RESULT_PRINT_FORMAT.format(**result) + "\n")
         pred_scenes.update(scenes)
 
     return pred_scenes
 
 
 def main(args):
-    pred_scenes = _detect_scenes(detector_type=args.detector, dataset=_make_dataset(args.dataset))
+    print(f"Evaluating {args.detector} on dataset {args.dataset}...\n")
+    pred_scenes = _detect_scenes(detector_type=args.detector, dataset=_DATASETS[args.dataset])
     result = Evaluator().evaluate_performance(pred_scenes)
-    print("Overall Results:")
-    print(
-        "Detector: {} Recall: {:.2f}, Precision: {:.2f}, F1: {:.2f} Elapsed time: {:.2f}".format(
-            args.detector, result["recall"], result["precision"], result["f1"], result["elapsed"]
-        )
-    )
+    print(f"\nOverall Results for {args.detector} on dataset {args.dataset}:")
+    print(_RESULT_PRINT_FORMAT.format(**result))
 
 
 if __name__ == "__main__":

benchmarks/autoshot_dataset.py → benchmark/autoshot_dataset.py

@@ -1,6 +1,7 @@
 import glob
 import os
 
+
 class AutoShotDataset:
     """
     The AutoShot Dataset (test splits) proposed by Zhu et al. in AutoShot: A Short Video Dataset and State-of-the-Art Shot Boundary Detection