diff --git a/benchmarks/BBC/.gitkeep b/benchmark/AutoShot/.gitkeep similarity index 100% rename from benchmarks/BBC/.gitkeep rename to benchmark/AutoShot/.gitkeep diff --git a/benchmark/BBC/.gitkeep b/benchmark/BBC/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/benchmarks/README.md b/benchmark/README.md similarity index 84% rename from benchmarks/README.md rename to benchmark/README.md index 2ad81c08..47777f29 100644 --- a/benchmarks/README.md +++ b/benchmark/README.md @@ -24,13 +24,13 @@ rm AutoShot.tar.gz ``` ## Evaluation -To evaluate PySceneDetect on a dataset, run the following command: +To evaluate PySceneDetect on a dataset, run the following command from the root of the repo: ``` -python benchmark.py -d --detector +python -m benchmark -d --detector ``` For example, to evaluate ContentDetector on the BBC dataset: ``` -python evaluate.py -d BBC --detector detect-content +python -m benchmark -d BBC --detector detect-content ``` ### Result @@ -40,10 +40,10 @@ The performance is computed as recall, precision, f1, and elapsed time. | Detector | Recall | Precision | F1 | Elapsed time (second) | |:-----------------:|:------:|:---------:|:-----:|:---------------------:| -| AdaptiveDetector | 87.52 | 97.21 | 92.11 | 27.84 | -| ContentDetector | 85.23 | 89.53 | 87.33 | 26.46 | -| HashDetector | 92.96 | 76.27 | 83.79 | 16.26 | -| HistogramDetector | 90.55 | 72.76 | 80.68 | 16.13 | +| AdaptiveDetector | 87.12 | 96.55 | 91.59 | 27.84 | +| ContentDetector | 84.70 | 88.77 | 86.69 | 28.20 | +| HashDetector | 92.30 | 75.56 | 83.10 | 16.00 | +| HistogramDetector | 89.84 | 72.03 | 79.96 | 15.13 | | ThresholdDetector | 0.00 | 0.00 | 0.00 | 18.95 | #### AutoShot diff --git a/benchmarks/benchmark.py b/benchmark/__main__.py similarity index 56% rename from benchmarks/benchmark.py rename to benchmark/__main__.py index c5c51376..b1a90d67 100644 --- a/benchmarks/benchmark.py +++ b/benchmark/__main__.py @@ -1,12 +1,12 @@ import argparse import time +import os -from bbc_dataset import BBCDataset -from autoshot_dataset import AutoShotDataset - -from evaluator import Evaluator from tqdm import tqdm +from benchmark.autoshot_dataset import AutoShotDataset +from benchmark.bbc_dataset import BBCDataset +from benchmark.evaluator import Evaluator from scenedetect import ( AdaptiveDetector, ContentDetector, @@ -18,22 +18,27 @@ def _make_detector(detector_name: str): - detector_map = { - "detect-adaptive": AdaptiveDetector(), - "detect-content": ContentDetector(), - "detect-hash": HashDetector(), - "detect-hist": HistogramDetector(), - "detect-threshold": ThresholdDetector(), - } - return detector_map[detector_name] + if detector_name == "detect-adaptive": + return AdaptiveDetector() + if detector_name == "detect-content": + return ContentDetector() + if detector_name == "detect-hash": + return HashDetector() + if detector_name == "detect-hist": + return HistogramDetector() + if detector_name == "detect-threshold": + return ThresholdDetector() + raise RuntimeError(f"Unknown detector: {detector_name}") + +_DATASETS = { + "BBC": BBCDataset("benchmark/BBC"), + "AutoShot": AutoShotDataset("benchmark/AutoShot"), +} -def _make_dataset(dataset_name: str): - dataset_map = { - "BBC": BBCDataset("BBC"), - "AutoShot": AutoShotDataset("AutoShot"), - } - return dataset_map[dataset_name] +_RESULT_PRINT_FORMAT = ( + "Recall: {recall:.2f}, Precision: {precision:.2f}, F1: {f1:.2f} Elapsed time: {elapsed:.2f}\n" +) def _detect_scenes(detector_type: str, dataset): @@ -43,34 +48,28 @@ def _detect_scenes(detector_type: str, dataset): detector = _make_detector(detector_type) pred_scene_list = detect(video_file, detector) elapsed = time.time() - start + filename = os.path.basename(video_file) scenes = { scene_file: { - "video_file": video_file, + "video_file": filename, "elapsed": elapsed, "pred_scenes": [scene[1].frame_num for scene in pred_scene_list], } } result = Evaluator().evaluate_performance(scenes) - print(f"{video_file} results:") - print( - "Recall: {:.2f}, Precision: {:.2f}, F1: {:.2f} Elapsed time: {:.2f}\n".format( - result["recall"], result["precision"], result["f1"], result["elapsed"] - ) - ) + print(f"\n{filename} results:") + print(_RESULT_PRINT_FORMAT.format(**result) + "\n") pred_scenes.update(scenes) return pred_scenes def main(args): - pred_scenes = _detect_scenes(detector_type=args.detector, dataset=_make_dataset(args.dataset)) + print(f"Evaluating {args.detector} on dataset {args.dataset}...\n") + pred_scenes = _detect_scenes(detector_type=args.detector, dataset=_DATASETS[args.dataset]) result = Evaluator().evaluate_performance(pred_scenes) - print("Overall Results:") - print( - "Detector: {} Recall: {:.2f}, Precision: {:.2f}, F1: {:.2f} Elapsed time: {:.2f}".format( - args.detector, result["recall"], result["precision"], result["f1"], result["elapsed"] - ) - ) + print(f"\nOverall Results for {args.detector} on dataset {args.dataset}:") + print(_RESULT_PRINT_FORMAT.format(**result)) if __name__ == "__main__": diff --git a/benchmarks/autoshot_dataset.py b/benchmark/autoshot_dataset.py similarity index 99% rename from benchmarks/autoshot_dataset.py rename to benchmark/autoshot_dataset.py index 41f86a17..5312704e 100644 --- a/benchmarks/autoshot_dataset.py +++ b/benchmark/autoshot_dataset.py @@ -1,6 +1,7 @@ import glob import os + class AutoShotDataset: """ The AutoShot Dataset (test splits) proposed by Zhu et al. in AutoShot: A Short Video Dataset and State-of-the-Art Shot Boundary Detection diff --git a/benchmarks/bbc_dataset.py b/benchmark/bbc_dataset.py similarity index 100% rename from benchmarks/bbc_dataset.py rename to benchmark/bbc_dataset.py diff --git a/benchmarks/evaluator.py b/benchmark/evaluator.py similarity index 100% rename from benchmarks/evaluator.py rename to benchmark/evaluator.py