Added initial speed benchmark script

README.md

@@ -0,0 +1,74 @@
+# speed_benchmark 
+
+Drive OpenALPR on all CPU cores to benchmark speed for various video resolutions
+
+## Prequisites
+
+* OpenALPR commercial license (2-week evaluation licenses can be obtained from 
+[here](https://license.openalpr.com/evalrequest/))
+* Ubuntu 18.04, Ubuntu 16.04, or Windows 10
+* Python3
+
+## Installation
+
+1. Download the OpenALPR [SDK](http://doc.openalpr.com/sdk.html#installation) 
+2. Clone this repository `git clone https://github.com/addisonklinke/openalpr-consulting.git`
+3. Install the Python requirements `pip install -r requirements.txt`
+
+## Usage
+
+1. View all command line options by running `python speed_benchmark.py -h`
+2. Select your desired resolution(s) and run a benchmark with 1 stream. Options are `vga, 720p, 1080p, and 4k`
+3. Check the average CPU utilization in the output. Resolutions with a utilization less than 95% are bottlenecked on
+decoding the video stream (typical for higher resolutions). These should be rerun with additional streams for a 
+better estimate of maximum performance using the `--streams` flag
+
+## Sample Output
+
+```commandline
+user@ubuntu:~/openalpr-consulting/speed-bench$ python speed_benchmark.py --streams 4
+Initializing...
+	Operating system: Linux
+	CPU model: Intel(R) Core(TM) i7-8750H CPU @ 2.20GHz
+	Runtime data: /usr/share/openalpr/runtime_data
+	OpenALPR configuration: /usr/share/openalpr/config/openalpr.defaults.conf
+Downloading benchmark videos...
+	Found local vga
+	Downloaded 720p
+	Found local 1080p
+	Found local 4k
+Processing vga...
+Processing 720p...
+Processing 1080p...
+Processing 4k...
++---------------------------------------------------------+
+|      OpenALPR Benchmark: 4 stream(s) on 12 threads      |
++------------+-----------+-----------+-----------+--------+
+| Resolution | Total FPS | CPU (Avg) | CPU (Max) | Frames |
++------------+-----------+-----------+-----------+--------+
+|    vga     |    89.7   |    98.6   |   100.0   | 10978  |
+|    720p    |    68.7   |    98.2   |   100.0   |  1125  |
+|   1080p    |    43.2   |    97.5   |   100.0   |  600   |
+|     4k     |    36.2   |    99.5   |   100.0   |  870   |
++------------+-----------+-----------+-----------+--------+
+```
+
+To estimate the number of cameras for a given total FPS value, use the following per-camera rules of thumb
+
+* **Low Speed** (under 25 mph): 5-10 fps
+* **Medium Speed** (25-45 mph): 10-15 fps
+* **High Speed** (over 45 mph): 15-30 fps
+
+## Running in Docker
+
+If preferred, you can install OpenALPR software in our pre-built Docker container
+
+```bash
+docker run -d -P -v openalpr-vol1-config:/etc/openalpr/ -v openalpr-vol1-images:/var/lib/openalpr/ -it openalpr/commercial-agent
+docker exec -it <container> /bin/bash
+apt update && apt install -y curl python-pip git
+git clone https://github.com/addisonklinke/openalpr-consulting.git
+cd openalpr-consulting/speed-bench
+pip install -r requirements.txt
+bash <(curl https://deb.openalpr.com/install)  # Select SDK
+```

requirements.txt

@@ -0,0 +1,3 @@
+psutil>=5.6.2
+PTable>=0.9.2
+statistics>=1.0.3.5

speed_benchmark.py

@@ -0,0 +1,219 @@
+import argparse
+from itertools import cycle
+from multiprocessing import cpu_count
+import os
+import platform
+import re
+from statistics import mean
+import subprocess
+from threading import Thread, Lock
+from time import time, sleep
+import urllib
+from prettytable import PrettyTable
+import psutil
+from alprstream import AlprStream
+from openalpr import Alpr
+from vehicleclassifier import VehicleClassifier
+
+
+def get_cpu_model(operating):
+    if operating == 'linux':
+        cpu_info = subprocess.check_output('lscpu').strip().decode().split('\n')
+        model_regex = re.compile('^Model name')
+        model = [c for c in cpu_info if model_regex.match(c)]
+        model = model[0].split(':')[-1].strip()
+    elif operating == 'windows':
+        model = platform.processor()
+    else:
+        raise ValueError('Expected OS to be linux or windows, but received {}'.format(operating))
+    return model
+
+
+class AlprBench:
+    """Benchmark OpenALPR software speed for various video resolutions.
+
+    :param int num_streams: Number of camera streams to simulate.
+    :param str or [str] resolution: Resolution(s) of videos to benchmark.
+    :param str downloads: Folder to save benchmark videos to.
+    :param str runtime: Path to runtime data folder.
+    :param str config: Path to OpenALPR configuration file.
+    :param bool quiet: Suppress all output besides final results.
+    """
+    def __init__(self, num_streams, resolution, downloads='/tmp/alprbench', runtime=None, config=None, quiet=False):
+
+        # Transfer parameters to attributes
+        self.quiet = quiet
+        self.message('Initializing...')
+        self.num_streams = num_streams
+        if isinstance(resolution, str):
+            if resolution == 'all':
+                self.resolution = ['vga', '720p', '1080p', '4k']
+            else:
+                self.resolution = [resolution]
+        elif isinstance(resolution, list):
+            self.resolution = resolution
+        else:
+            raise ValueError('Expected list or str for resolution, but received {}'.format(resolution))
+        self.downloads = downloads
+        if not os.path.exists(self.downloads):
+            os.mkdir(self.downloads)
+
+        # Prepare other attributes
+        self.cpu_usage = {r: [] for r in self.resolution}
+        self.threads_active = False
+        self.frame_counter = 0
+        self.mutex = Lock()
+        self.streams = []
+        self.round_robin = cycle(range(self.num_streams))
+        self.results = PrettyTable()
+        self.results.field_names = ['Resolution', 'Total FPS', 'CPU (Avg)', 'CPU (Max)', 'Frames']
+        self.results.title = 'OpenALPR Speed: {} stream(s) on {} threads'.format(
+            self.num_streams, cpu_count())
+
+        # Detect operating system
+        if platform.system().lower().find('linux') == 0:
+            operating = 'linux'
+            self.message('\tOperating system: Linux')
+            self.message('\tCPU model: {}'.format(get_cpu_model('linux')))
+        elif platform.system().lower().find('windows') == 0:
+            operating = 'windows'
+            self.message('\tOperating system: Windows')
+            self.message('\tCPU model: {}'.format(get_cpu_model('windows')))
+        else:
+            raise OSError('Detected OS other than Linux or Windows')
+
+        # Define default runtime and config paths if not specified
+        if runtime is None:
+            self.runtime = '/usr/share/openalpr/runtime_data'
+            if operating == 'windows':
+                self.runtime = 'C:/OpenALPR/Agent' + self.runtime
+        if config is None:
+            self.config = '/usr/share/openalpr/config/openalpr.defaults.conf'
+            if operating == 'windows':
+                self.config = 'C:/OpenALPR/Agent' + self.config
+        self.message('\tRuntime data: {}'.format(self.runtime))
+        self.message('\tOpenALPR configuration: {}'.format(self.config))
+
+    def __call__(self):
+        """Run threaded benchmarks on all requested resolutions."""
+        videos = self.download_benchmarks()
+        self.streams = [AlprStream(10, False) for _ in range(self.num_streams)]
+        name_regex = re.compile('(?<=\/)[^\.\/]+')
+        self.threads_active = True
+
+        for v in videos:
+            res = name_regex.findall(v)[-1]
+            self.message('Processing {}...'.format(res))
+            self.frame_counter = 0
+            threads = []
+            for s in self.streams:
+                s.connect_video_file(v, 0)
+            for i in range(cpu_count()):
+                threads.append(Thread(target=self.worker, args=(res, )))
+                threads[i].setDaemon(True)
+            start = time()
+            for t in threads:
+                t.start()
+            while len(threads) > 0:
+                try:
+                    threads = [t.join() for t in threads if t is not None and t.isAlive()]
+                except KeyboardInterrupt:
+                    print('\n\nCtrl-C received! Sending kill to threads...')
+                    self.threads_active = False
+                    break
+            elapsed = time() - start
+            self.format_results(res, elapsed)
+        print(self.results)
+
+    def download_benchmarks(self):
+        """Save requested benchmark videos locally.
+
+        :return [str] videos: Filepaths to downloaded videos.
+        """
+        videos = []
+        endpoint = 'http://download.openalpr.com/bench'
+        files = ['vga.webm', '720p.mp4', '1080p.mp4', '4k.mp4']
+        existing = os.listdir(self.downloads)
+        self.message('Downloading benchmark videos...')
+        for f in files:
+            res = f.split('.')[0]
+            if res in self.resolution:
+                out = os.path.join(self.downloads, f)
+                videos.append(out)
+                if f not in existing:
+                    _ = urllib.urlretrieve(os.path.join(endpoint, f), out)
+                    self.message('\tDownloaded {}'.format(res))
+                else:
+                    self.message('\tFound local {}'.format(res))
+        return videos
+
+    def format_results(self, resolution, elapsed):
+        """Update results table.
+
+        :param str resolution: Resolution of the video that was benchmarked.
+        :param float elapsed: Time to process video (in seconds).
+        :return: None
+        """
+        total_fps = '{:.1f}'.format(self.frame_counter / elapsed)
+        avg_cpu = '{:.1f}'.format(mean(self.cpu_usage[resolution]))
+        max_cpu = '{:.1f}'.format(max(self.cpu_usage[resolution]))
+        avg_frames = int(self.frame_counter / self.num_streams)
+        self.results.add_row([resolution, total_fps, avg_cpu, max_cpu, avg_frames])
+
+    def message(self, msg):
+        """Control verbosity of output.
+
+        :param str msg: Message to display.
+        :return: None
+        """
+        if not self.quiet:
+            print(msg)
+
+    def worker(self, resolution):
+        """Thread for a single Alpr and VehicleClassifier instance."""
+        alpr = Alpr('us', self.config, self.runtime)
+        vehicle = VehicleClassifier(self.config, self.runtime)
+        active_streams = sum([s.video_file_active() for s in self.streams])
+        total_queue = sum([s.get_queue_size() for s in self.streams])
+        while active_streams or total_queue > 0:
+            if not self.threads_active:
+                break
+            active_streams = sum([s.video_file_active() for s in self.streams])
+            total_queue = sum([s.get_queue_size() for s in self.streams])
+            idx = next(self.round_robin)
+            if self.streams[idx].get_queue_size() == 0:
+                sleep(0.1)
+                continue
+            results = self.streams[idx].process_frame(alpr)
+            if results['epoch_time'] > 0 and results['processing_time_ms'] > 0:
+                _ = self.streams[idx].pop_completed_groups_and_recognize_vehicle(vehicle)
+                self.mutex.acquire()
+                self.frame_counter += 1
+                if self.frame_counter % 10 == 0:
+                    self.cpu_usage[resolution].append(psutil.cpu_percent())
+                self.mutex.release()
+
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(
+        description='Benchmark OpenALPR software speed at various video resolutions',
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    parser.add_argument('-d', '--download_dir', type=str, default='/tmp/alprbench', help='folder to save videos')
+    parser.add_argument('-q', '--quiet', action='store_true', help='suppress all output besides final results')
+    parser.add_argument('-r', '--resolution', type=str, default='all', help='video resolution to benchmark on')
+    parser.add_argument('-s', '--streams', type=int, default=1, help='number of camera streams to simulate')
+    parser.add_argument('--config', type=str, help='path to OpenALPR config, detects Windows/Linux and uses defaults')
+    parser.add_argument('--runtime', type=str, help='path to runtime data, detects Windows/Linux and uses defaults')
+    args = parser.parse_args()
+
+    if ',' in args.resolution:
+        args.resolution = [r.strip() for r in args.resolution.split(',')]
+    bench = AlprBench(
+        args.streams,
+        args.resolution,
+        args.download_dir,
+        args.runtime,
+        args.config,
+        args.quiet)
+    bench()