Merge pull request #6 from arsfutura/develop

v0.4.0

Former-commit-id: 3ca9a35

LICENSE

@@ -0,0 +1,29 @@
+BSD 3-Clause License
+
+Copyright (c) 2019, Ars Futura
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

README.md

@@ -1,14 +1,17 @@
-This repository provides a framework for creating and using a Face Recognition system. 
+# Framework for creating and using Face Recognition system.
+This repository provides a simple framework for creating and using Face Recognition system. There is also a 
+[blog post](https://arsfutura.co/magazine/face-recognition-with-facenet-and-mtcnn/) associated with this repository 
+which gives more details about the framework.  
 
-![Face Recognition illustration](readme-illustration.png)
+![Face Recognition illustration](images/readme-illustration.png)
 
 # Installation
-Make sure you have [Python 3](https://realpython.com/installing-python/) and 
+Make sure you have [Python 3.5+](https://realpython.com/installing-python/) and 
 [`pip`](https://www.makeuseof.com/tag/install-pip-for-python/) installed.
 
 Install dependencies
 ```
-pip install -r requirements.txt -f https://download.pytorch.org/whl/torch_stable.html
+pip install -r requirements.txt
 ```
 
 # Train the Face Recognition system
@@ -36,6 +39,35 @@ After preparing the images run the following command to train the Face Recogniti
 ``` 
 The previous command will generate `model/face_recogniser.pkl` which represents the trained Face Recognition system.
 
+`train.py` has other options for training too. Slow part of training is generating embeddings from images. You could 
+pre-generate embeddings with `util/generate_embeddings.py` and then just forward path to embeddings to train script, 
+that would speed up experimenting with training a lot.
+
+```
+usage: train.py [-h] [-d DATASET_PATH] [-e EMBEDDINGS_PATH] [-l LABELS_PATH]
+                [-c CLASS_TO_IDX_PATH] [--grid-search]
+
+Script for training Face Recognition model. You can either give path to
+dataset or provide path to pre-generated embeddings, labels and class_to_idx.
+You can pre-generate this with util/generate_embeddings.py script.
+
+optional arguments:
+  -h, --help            show this help message and exit
+  -d DATASET_PATH, --dataset-path DATASET_PATH
+                        Path to folder with images.
+  -e EMBEDDINGS_PATH, --embeddings-path EMBEDDINGS_PATH
+                        Path to file with embeddings.
+  -l LABELS_PATH, --labels-path LABELS_PATH
+                        Path to file with labels.
+  -c CLASS_TO_IDX_PATH, --class-to-idx-path CLASS_TO_IDX_PATH
+                        Path to pickled class_to_idx dict.
+  --grid-search         If this option is enabled, grid search will be
+                        performed to estimate C parameter of Logistic
+                        Regression classifier. In order to use this option you
+                        have to have at least 3 examples of every class in
+                        your dataset. It is recommended to enable this option.
+```
+
 # Using Face Recognition
 
 After training the Face Recognition system you can use it in several ways. You can use one of the inference scripts or via a REST API. 
@@ -77,9 +109,14 @@ Video stream example:
 You can use the trained Face Recognition system as a REST API. The `api` folder contains a simple 
 [Flask](https://palletsprojects.com/p/flask/) API which provides frontend for the Face Recognition system.
 
-Run the server using the following command:
+Run the development server using the following command:
+```
+tasks/run_dev_server.sh
+```
+
+Run the production server using the following command:
 ```
-tasks/run_server.sh
+tasks/run_prod_server.sh
 ```
 
 The server is running on port `5000`.

api/Dockerfile

@@ -4,17 +4,19 @@ MAINTAINER Luka Dulčić "culuma@arsfutura.co"
 
 RUN mkdir -p /app && \
     apt-get update -y && \
-    apt-get install -y libsm6 libxext6 libxrender-dev libglib2.0-0
+    apt-get install -y build-essential python3-dev libsm6 libxext6 libxrender-dev libglib2.0-0
 
 WORKDIR /app
 
 # We copy just the requirements.txt first to leverage Docker cache
 COPY ./requirements.txt /app/requirements.txt
 
-RUN pip3 install -r requirements.txt -f https://download.pytorch.org/whl/torch_stable.html
+RUN pip3 install -r requirements.txt
 
 COPY face_recognition /app/face_recognition
 COPY model /app/model
 COPY api /app/api
+COPY tasks/run_prod_server.sh /app/run_prod_server.sh
+RUN chmod +x run_prod_server.sh
 
-CMD [ "python3", "-m", "api.app" ]
+CMD [ "./run_prod_server.sh" ]

api/app.py

@@ -4,8 +4,10 @@
 from flask import Flask
 from flask_restplus import Api, Resource, fields, abort, inputs
 from werkzeug.datastructures import FileStorage
+from face_recognition import preprocessing
 
 face_recogniser = joblib.load('model/face_recogniser.pkl')
+preprocess = preprocessing.ExifOrientationNormalize()
 
 IMAGE_KEY = 'image'
 INCLUDE_PREDICTIONS_KEY = 'include_predictions'
@@ -57,6 +59,9 @@ def post(self):
             abort(400, "Image field '{}' doesn't exist in request!".format(IMAGE_KEY))
 
         img = Image.open(io.BytesIO(args[IMAGE_KEY].read()))
+        img = preprocess(img)
+        # convert image to RGB (stripping alpha channel if exists)
+        img = img.convert('RGB')
         faces = face_recogniser(img)
         return \
             {

face_recognition/face_features_extractor.py

@@ -7,7 +7,6 @@
 
 class FaceFeaturesExtractor:
     def __init__(self):
-        # TODO adjust threshold for detecting face
         self.aligner = MTCNN(prewhiten=False, keep_all=True, thresholds=[0.6, 0.7, 0.9])
         self.facenet_preprocess = transforms.Compose([preprocessing.Whitening()])
         self.facenet = InceptionResnetV1(pretrained='vggface2').eval()

face_recognition/preprocessing.py

@@ -21,7 +21,7 @@ class ExifOrientationNormalize(object):
     """
 
     def __call__(self, img):
-        if 'parsed_exif' in img.info:
+        if 'parsed_exif' in img.info and exif_orientation_tag in img.info['parsed_exif']:
             orientation = img.info['parsed_exif'][exif_orientation_tag]
             transposes = exif_transpose_sequences[orientation]
             for trans in transposes:

inference/classifier.py

@@ -6,6 +6,7 @@
 from PIL import Image
 from .util import draw_bb_on_img
 from .constants import MODEL_PATH
+from face_recognition import preprocessing
 
 
 def parse_args():
@@ -26,8 +27,10 @@ def recognise_faces(img):
 
 def main():
     args = parse_args()
+    preprocess = preprocessing.ExifOrientationNormalize()
     img = Image.open(args.image_path)
     filename = img.filename
+    img = preprocess(img)
     img = img.convert('RGB')
 
     faces, img = recognise_faces(img)

inference/video_classifier.py

@@ -4,21 +4,23 @@
 import cv2
 import numpy as np
 from PIL import Image
+from face_recognition import preprocessing
 from .util import draw_bb_on_img
 from .constants import MODEL_PATH
 
 
 def main():
     cap = cv2.VideoCapture(0)
     face_recogniser = joblib.load(MODEL_PATH)
+    preprocess = preprocessing.ExifOrientationNormalize()
 
     while True:
         # Capture frame-by-frame
         ret, frame = cap.read()
         frame = cv2.flip(frame, 1)
 
         img = Image.fromarray(frame)
-        faces = face_recogniser(img)
+        faces = face_recogniser(preprocess(img))
         if faces is not None:
             draw_bb_on_img(faces, img)
 

requirements.txt

@@ -5,10 +5,11 @@ facenet_pytorch==0.1.0
 Flask==1.1.1
 flask_restplus==0.13.0
 joblib==0.13.2
-matplotlib==3.1.1
+matplotlib==3.0.0
 seaborn==0.9.0
 scikit_learn==0.21.3
-torch==1.2.0+cpu
-torchvision==0.4.0+cpu
+torch==1.2.0
+torchvision==0.4.0
 Werkzeug==0.15.2
 opencv-python==4.1.0.25
+uWSGI==2.0.18

tasks/run_prod_server.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+uwsgi --http 0.0.0.0:5000 --wsgi-file api/app.py --callable app --processes 5 --threads 2

training/train.py

@@ -5,23 +5,34 @@
 from PIL import Image
 from torchvision import transforms, datasets
 from sklearn.linear_model import LogisticRegression
+from sklearn.model_selection import GridSearchCV
+from sklearn import metrics
 from face_recognition import preprocessing, FaceFeaturesExtractor, FaceRecogniser
 
-
 MODEL_DIR_PATH = 'model'
 
 
 def parse_args():
-    parser = argparse.ArgumentParser()
-    parser.add_argument('-d', '--dataset-path', required=True, help='Path to folder with images.')
+    parser = argparse.ArgumentParser(
+        description='Script for training Face Recognition model. You can either give path to dataset or provide path '
+                    'to pre-generated embeddings, labels and class_to_idx. You can pre-generate this with '
+                    'util/generate_embeddings.py script.')
+    parser.add_argument('-d', '--dataset-path', help='Path to folder with images.')
+    parser.add_argument('-e', '--embeddings-path', help='Path to file with embeddings.')
+    parser.add_argument('-l', '--labels-path', help='Path to file with labels.')
+    parser.add_argument('-c', '--class-to-idx-path', help='Path to pickled class_to_idx dict.')
+    parser.add_argument('--grid-search', action='store_true',
+                        help='If this option is enabled, grid search will be performed to estimate C parameter of '
+                             'Logistic Regression classifier. In order to use this option you have to have at least '
+                             '3 examples of every class in your dataset. It is recommended to enable this option.')
     return parser.parse_args()
 
 
 def dataset_to_embeddings(dataset, features_extractor):
     transform = transforms.Compose([
-            preprocessing.ExifOrientationNormalize(),
-            transforms.Resize(1024)
-        ])
+        preprocessing.ExifOrientationNormalize(),
+        transforms.Resize(1024)
+    ])
 
     embeddings = []
     labels = []
@@ -40,17 +51,43 @@ def dataset_to_embeddings(dataset, features_extractor):
     return np.stack(embeddings), labels
 
 
+def load_data(args, features_extractor):
+    if args.embeddings_path:
+        return np.loadtxt(args.embeddings_path), \
+               np.loadtxt(args.labels_path, dtype='str').tolist(), \
+               joblib.load(args.class_to_idx_path)
+
+    dataset = datasets.ImageFolder(args.dataset_path)
+    embeddings, labels = dataset_to_embeddings(dataset, features_extractor)
+    return embeddings, labels, dataset.class_to_idx
+
+
+def train(args, embeddings, labels):
+    softmax = LogisticRegression(solver='lbfgs', multi_class='multinomial', C=10, max_iter=10000)
+    if args.grid_search:
+        clf = GridSearchCV(
+            estimator=softmax,
+            param_grid={'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000]},
+            cv=3
+        )
+    else:
+        clf = softmax
+    clf.fit(embeddings, labels)
+
+    return clf.best_estimator_ if args.grid_search else clf
+
+
 def main():
     args = parse_args()
 
     features_extractor = FaceFeaturesExtractor()
-    dataset = datasets.ImageFolder(args.dataset_path)
-    embeddings, labels = dataset_to_embeddings(dataset, features_extractor)
+    embeddings, labels, class_to_idx = load_data(args, features_extractor)
+    clf = train(args, embeddings, labels)
 
-    clf = LogisticRegression(C=10, solver='lbfgs', multi_class='multinomial')
-    clf.fit(embeddings, labels)
+    idx_to_class = {v: k for k, v in class_to_idx.items()}
 
-    idx_to_class = {v: k for k, v in dataset.class_to_idx.items()}
+    target_names = map(lambda i: i[1], sorted(idx_to_class.items(), key=lambda i: i[0]))
+    print(metrics.classification_report(labels, clf.predict(embeddings), target_names=list(target_names)))
 
     if not os.path.isdir(MODEL_DIR_PATH):
         os.mkdir(MODEL_DIR_PATH)

util/collect_face_images.py

@@ -15,7 +15,7 @@ def main(directory, name, test):
         # Display the resulting frame
         cv2.imshow(name, frame)
         if not test and i != 0 and i % 10 == 0:
-            cv2.imwrite("{}/{}{}.png".format(directory, name, i / 10), frame)
+            cv2.imwrite("{}/{}{}.png".format(directory, name, int(i / 10)), frame)
         i += 1
         if cv2.waitKey(1) & 0xFF == ord('q'):
             break

util/generate_embeddings.py

@@ -1,5 +1,6 @@
 import argparse
 import os
+import joblib
 import numpy as np
 import torch
 from torchvision import datasets
@@ -10,14 +11,26 @@
 def parse_args():
     parser = argparse.ArgumentParser(
         "Script for generating face embeddings. Output of this script is 'embeddings.txt' which contains embeddings "
-        "for all input images and 'labels.txt' which contains label for every embedding.")
+        "for all input images, 'labels.txt' which contains label for every embedding and 'class_to_idx.pkl' which "
+        "is serializes dictionary which maps classes to its index.")
     parser.add_argument('--input-folder', required=True,
                         help='Root folder where images are. This folder contains sub-folders for each class.')
     parser.add_argument('--output-folder', required=True,
                         help='Output folder where image embeddings and labels will be saved.')
     return parser.parse_args()
 
 
+def normalise_string(string):
+    return string.lower().replace(' ', '_')
+
+
+def normalise_dict_keys(dictionary):
+    new_dict = dict()
+    for key in dictionary.keys():
+        new_dict[normalise_string(key)] = dictionary[key]
+    return new_dict
+
+
 def main():
     torch.set_grad_enabled(False)
     args = parse_args()
@@ -26,11 +39,13 @@ def main():
     dataset = datasets.ImageFolder(args.input_folder)
     embeddings, labels = dataset_to_embeddings(dataset, features_extractor)
 
+    dataset.class_to_idx = normalise_dict_keys(dataset.class_to_idx)
     idx_to_class = {v: k for k, v in dataset.class_to_idx.items()}
     labels = list(map(lambda idx: idx_to_class[idx], labels))
 
     np.savetxt(args.output_folder + os.path.sep + 'embeddings.txt', embeddings)
     np.savetxt(args.output_folder + os.path.sep + 'labels.txt', np.array(labels, dtype=np.str).reshape(-1, 1), fmt="%s")
+    joblib.dump(dataset.class_to_idx, args.output_folder + os.path.sep + 'class_to_idx.pkl')
 
 
 if __name__ == '__main__':