diff --git a/README.md b/README.md new file mode 100644 index 0000000..83c5629 --- /dev/null +++ b/README.md @@ -0,0 +1,74 @@ +# speed_benchmark + +Drive OpenALPR on all CPU cores to benchmark speed for various video resolutions + +## Prequisites + +* OpenALPR commercial license (2-week evaluation licenses can be obtained from +[here](https://license.openalpr.com/evalrequest/)) +* Ubuntu 18.04, Ubuntu 16.04, or Windows 10 +* Python3 + +## Installation + +1. Download the OpenALPR [SDK](http://doc.openalpr.com/sdk.html#installation) +2. Clone this repository `git clone https://github.com/addisonklinke/openalpr-consulting.git` +3. Install the Python requirements `pip install -r requirements.txt` + +## Usage + +1. View all command line options by running `python speed_benchmark.py -h` +2. Select your desired resolution(s) and run a benchmark with 1 stream. Options are `vga, 720p, 1080p, and 4k` +3. Check the average CPU utilization in the output. Resolutions with a utilization less than 95% are bottlenecked on +decoding the video stream (typical for higher resolutions). These should be rerun with additional streams for a +better estimate of maximum performance using the `--streams` flag + +## Sample Output + +```commandline +user@ubuntu:~/openalpr-consulting/speed-bench$ python speed_benchmark.py --streams 4 +Initializing... + Operating system: Linux + CPU model: Intel(R) Core(TM) i7-8750H CPU @ 2.20GHz + Runtime data: /usr/share/openalpr/runtime_data + OpenALPR configuration: /usr/share/openalpr/config/openalpr.defaults.conf +Downloading benchmark videos... + Found local vga + Downloaded 720p + Found local 1080p + Found local 4k +Processing vga... +Processing 720p... +Processing 1080p... +Processing 4k... ++---------------------------------------------------------+ +| OpenALPR Benchmark: 4 stream(s) on 12 threads | ++------------+-----------+-----------+-----------+--------+ +| Resolution | Total FPS | CPU (Avg) | CPU (Max) | Frames | ++------------+-----------+-----------+-----------+--------+ +| vga | 89.7 | 98.6 | 100.0 | 10978 | +| 720p | 68.7 | 98.2 | 100.0 | 1125 | +| 1080p | 43.2 | 97.5 | 100.0 | 600 | +| 4k | 36.2 | 99.5 | 100.0 | 870 | ++------------+-----------+-----------+-----------+--------+ +``` + +To estimate the number of cameras for a given total FPS value, use the following per-camera rules of thumb + +* **Low Speed** (under 25 mph): 5-10 fps +* **Medium Speed** (25-45 mph): 10-15 fps +* **High Speed** (over 45 mph): 15-30 fps + +## Running in Docker + +If preferred, you can install OpenALPR software in our pre-built Docker container + +```bash +docker run -d -P -v openalpr-vol1-config:/etc/openalpr/ -v openalpr-vol1-images:/var/lib/openalpr/ -it openalpr/commercial-agent +docker exec -it <container> /bin/bash +apt update && apt install -y curl python-pip git +git clone https://github.com/addisonklinke/openalpr-consulting.git +cd openalpr-consulting/speed-bench +pip install -r requirements.txt +bash <(curl https://deb.openalpr.com/install) # Select SDK +``` diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..53e8327 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +psutil>=5.6.2 +PTable>=0.9.2 +statistics>=1.0.3.5 diff --git a/speed_benchmark.py b/speed_benchmark.py new file mode 100644 index 0000000..9b16b4c --- /dev/null +++ b/speed_benchmark.py @@ -0,0 +1,219 @@ +import argparse +from itertools import cycle +from multiprocessing import cpu_count +import os +import platform +import re +from statistics import mean +import subprocess +from threading import Thread, Lock +from time import time, sleep +import urllib +from prettytable import PrettyTable +import psutil +from alprstream import AlprStream +from openalpr import Alpr +from vehicleclassifier import VehicleClassifier + + +def get_cpu_model(operating): + if operating == 'linux': + cpu_info = subprocess.check_output('lscpu').strip().decode().split('\n') + model_regex = re.compile('^Model name') + model = [c for c in cpu_info if model_regex.match(c)] + model = model[0].split(':')[-1].strip() + elif operating == 'windows': + model = platform.processor() + else: + raise ValueError('Expected OS to be linux or windows, but received {}'.format(operating)) + return model + + +class AlprBench: + """Benchmark OpenALPR software speed for various video resolutions. + + :param int num_streams: Number of camera streams to simulate. + :param str or [str] resolution: Resolution(s) of videos to benchmark. + :param str downloads: Folder to save benchmark videos to. + :param str runtime: Path to runtime data folder. + :param str config: Path to OpenALPR configuration file. + :param bool quiet: Suppress all output besides final results. + """ + def __init__(self, num_streams, resolution, downloads='/tmp/alprbench', runtime=None, config=None, quiet=False): + + # Transfer parameters to attributes + self.quiet = quiet + self.message('Initializing...') + self.num_streams = num_streams + if isinstance(resolution, str): + if resolution == 'all': + self.resolution = ['vga', '720p', '1080p', '4k'] + else: + self.resolution = [resolution] + elif isinstance(resolution, list): + self.resolution = resolution + else: + raise ValueError('Expected list or str for resolution, but received {}'.format(resolution)) + self.downloads = downloads + if not os.path.exists(self.downloads): + os.mkdir(self.downloads) + + # Prepare other attributes + self.cpu_usage = {r: [] for r in self.resolution} + self.threads_active = False + self.frame_counter = 0 + self.mutex = Lock() + self.streams = [] + self.round_robin = cycle(range(self.num_streams)) + self.results = PrettyTable() + self.results.field_names = ['Resolution', 'Total FPS', 'CPU (Avg)', 'CPU (Max)', 'Frames'] + self.results.title = 'OpenALPR Speed: {} stream(s) on {} threads'.format( + self.num_streams, cpu_count()) + + # Detect operating system + if platform.system().lower().find('linux') == 0: + operating = 'linux' + self.message('\tOperating system: Linux') + self.message('\tCPU model: {}'.format(get_cpu_model('linux'))) + elif platform.system().lower().find('windows') == 0: + operating = 'windows' + self.message('\tOperating system: Windows') + self.message('\tCPU model: {}'.format(get_cpu_model('windows'))) + else: + raise OSError('Detected OS other than Linux or Windows') + + # Define default runtime and config paths if not specified + if runtime is None: + self.runtime = '/usr/share/openalpr/runtime_data' + if operating == 'windows': + self.runtime = 'C:/OpenALPR/Agent' + self.runtime + if config is None: + self.config = '/usr/share/openalpr/config/openalpr.defaults.conf' + if operating == 'windows': + self.config = 'C:/OpenALPR/Agent' + self.config + self.message('\tRuntime data: {}'.format(self.runtime)) + self.message('\tOpenALPR configuration: {}'.format(self.config)) + + def __call__(self): + """Run threaded benchmarks on all requested resolutions.""" + videos = self.download_benchmarks() + self.streams = [AlprStream(10, False) for _ in range(self.num_streams)] + name_regex = re.compile('(?<=\/)[^\.\/]+') + self.threads_active = True + + for v in videos: + res = name_regex.findall(v)[-1] + self.message('Processing {}...'.format(res)) + self.frame_counter = 0 + threads = [] + for s in self.streams: + s.connect_video_file(v, 0) + for i in range(cpu_count()): + threads.append(Thread(target=self.worker, args=(res, ))) + threads[i].setDaemon(True) + start = time() + for t in threads: + t.start() + while len(threads) > 0: + try: + threads = [t.join() for t in threads if t is not None and t.isAlive()] + except KeyboardInterrupt: + print('\n\nCtrl-C received! Sending kill to threads...') + self.threads_active = False + break + elapsed = time() - start + self.format_results(res, elapsed) + print(self.results) + + def download_benchmarks(self): + """Save requested benchmark videos locally. + + :return [str] videos: Filepaths to downloaded videos. + """ + videos = [] + endpoint = 'http://download.openalpr.com/bench' + files = ['vga.webm', '720p.mp4', '1080p.mp4', '4k.mp4'] + existing = os.listdir(self.downloads) + self.message('Downloading benchmark videos...') + for f in files: + res = f.split('.')[0] + if res in self.resolution: + out = os.path.join(self.downloads, f) + videos.append(out) + if f not in existing: + _ = urllib.urlretrieve(os.path.join(endpoint, f), out) + self.message('\tDownloaded {}'.format(res)) + else: + self.message('\tFound local {}'.format(res)) + return videos + + def format_results(self, resolution, elapsed): + """Update results table. + + :param str resolution: Resolution of the video that was benchmarked. + :param float elapsed: Time to process video (in seconds). + :return: None + """ + total_fps = '{:.1f}'.format(self.frame_counter / elapsed) + avg_cpu = '{:.1f}'.format(mean(self.cpu_usage[resolution])) + max_cpu = '{:.1f}'.format(max(self.cpu_usage[resolution])) + avg_frames = int(self.frame_counter / self.num_streams) + self.results.add_row([resolution, total_fps, avg_cpu, max_cpu, avg_frames]) + + def message(self, msg): + """Control verbosity of output. + + :param str msg: Message to display. + :return: None + """ + if not self.quiet: + print(msg) + + def worker(self, resolution): + """Thread for a single Alpr and VehicleClassifier instance.""" + alpr = Alpr('us', self.config, self.runtime) + vehicle = VehicleClassifier(self.config, self.runtime) + active_streams = sum([s.video_file_active() for s in self.streams]) + total_queue = sum([s.get_queue_size() for s in self.streams]) + while active_streams or total_queue > 0: + if not self.threads_active: + break + active_streams = sum([s.video_file_active() for s in self.streams]) + total_queue = sum([s.get_queue_size() for s in self.streams]) + idx = next(self.round_robin) + if self.streams[idx].get_queue_size() == 0: + sleep(0.1) + continue + results = self.streams[idx].process_frame(alpr) + if results['epoch_time'] > 0 and results['processing_time_ms'] > 0: + _ = self.streams[idx].pop_completed_groups_and_recognize_vehicle(vehicle) + self.mutex.acquire() + self.frame_counter += 1 + if self.frame_counter % 10 == 0: + self.cpu_usage[resolution].append(psutil.cpu_percent()) + self.mutex.release() + + +if __name__ == '__main__': + + parser = argparse.ArgumentParser( + description='Benchmark OpenALPR software speed at various video resolutions', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument('-d', '--download_dir', type=str, default='/tmp/alprbench', help='folder to save videos') + parser.add_argument('-q', '--quiet', action='store_true', help='suppress all output besides final results') + parser.add_argument('-r', '--resolution', type=str, default='all', help='video resolution to benchmark on') + parser.add_argument('-s', '--streams', type=int, default=1, help='number of camera streams to simulate') + parser.add_argument('--config', type=str, help='path to OpenALPR config, detects Windows/Linux and uses defaults') + parser.add_argument('--runtime', type=str, help='path to runtime data, detects Windows/Linux and uses defaults') + args = parser.parse_args() + + if ',' in args.resolution: + args.resolution = [r.strip() for r in args.resolution.split(',')] + bench = AlprBench( + args.streams, + args.resolution, + args.download_dir, + args.runtime, + args.config, + args.quiet) + bench()