first commit

.gitignore

@@ -0,0 +1,3 @@
+*.mlmodel
+opencv2.xcframework
+.DS_Store

PoseDecoder/.gitignore

@@ -0,0 +1,7 @@
+.DS_Store
+/.build
+/Packages
+/*.xcodeproj
+xcuserdata/
+DerivedData/
+.swiftpm/xcode/package.xcworkspace/contents.xcworkspacedata

PoseDecoder/.swiftpm/xcode/package.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>IDEDidComputeMac32BitWarning</key>
+	<true/>
+</dict>
+</plist>

PoseDecoder/Package.swift

@@ -0,0 +1,41 @@
+// swift-tools-version:5.5
+// The swift-tools-version declares the minimum version of Swift required to build this package.
+
+import PackageDescription
+
+let package = Package(
+    name: "PoseDecoder",
+    products: [
+        // Products define the executables and libraries a package produces, and make them visible to other packages.
+        .library(
+            name: "PoseDecoder",
+            targets: ["PoseDecoder"]),
+        .library(
+            name: "PoseDecoderCPP",
+            targets: ["PoseDecoderCPP"]),
+    ],
+    dependencies: [
+        // Dependencies declare other packages that this package depends on.
+        // .package(url: /* package url */, from: "1.0.0"),
+    ],
+    targets: [
+        // Targets are the basic building blocks of a package. A target can define a module or a test suite.
+        // Targets can depend on other targets in this package, and on products in packages this package depends on.
+        .binaryTarget(
+            name: "opencv2",
+            path: "Frameworks/opencv2.xcframework"),
+        .target(
+            name: "PoseDecoderCPP",
+            dependencies: ["opencv2"],
+            path: "Sources/PoseDecoderCPP",
+            publicHeadersPath: "include"
+        ),
+        .target(
+            name: "PoseDecoder",
+            dependencies: ["PoseDecoderCPP", "opencv2"],
+            path: "Sources/PoseDecoder",
+            publicHeadersPath: "include"
+        ),
+    ],
+    cxxLanguageStandard: .cxx14
+)

PoseDecoder/README.md

@@ -0,0 +1,3 @@
+# PoseDecoder
+
+A description of this package.

PoseDecoder/Sources/PoseDecoder/PoseDecoder.mm

@@ -0,0 +1,189 @@
+#import <opencv2/opencv.hpp>
+#import <opencv2/imgcodecs/ios.h>
+#include <iostream>
+#include <vector>
+
+#import "PoseDecoder.h"
+#import "hpe_model_openpose.h"
+#import "utils/image_utils.h"
+#import <mach/mach_time.h>
+#import <CoreML/MLModel.h>
+#import <Vision/Vision.h>
+
+@implementation PoseDecoder
+
+-(UIImage*) detection: (float *) data
+             dataSize: (int) dataSize
+            uiImage: (UIImage*) uiImage
+           modelWidth: (int) modelWidth
+          modelHeight: (int) modelHeight
+                scale: (CGPoint) scale
+{
+    uint64_t start, elapsed;
+    start = mach_absolute_time();
+
+    int imgW = uiImage.size.width;
+    int imgH = uiImage.size.height;
+    int dim2 = modelHeight / 8;
+    int dim3 = modelWidth / 8;
+
+    int keypointsNumber = HPEOpenPose::keypointsNumber;
+
+    std::cout << "imgSize: " << imgW << "x" << imgH << std::endl;
+    std::cout << "keypoint: " << keypointsNumber << std::endl;
+
+    int mapIdxOffset = keypointsNumber + 1;
+    std::vector<float> vec(&data[0], data + dataSize);
+    std::vector<float>
+                hvec(vec.begin(), vec.begin() + mapIdxOffset*dim2*dim3),
+                pvec(vec.begin() + mapIdxOffset*dim2*dim3, vec.end());
+
+    std::cout << "hvec: " << hvec.size() << std::endl;
+    std::cout << "matrix: " << mapIdxOffset << "x" << dim2 << "x" << dim3 << std::endl;
+
+    float* predictions = &pvec[0];
+    float* heats = &hvec[0];
+
+    std::vector<cv::Mat> heatMaps(mapIdxOffset);
+    for (size_t i = 0; i < heatMaps.size(); i++) {
+        heatMaps[i] = cv::Mat(dim2, dim3, CV_32FC1, heats + i * dim2 * dim3);
+    }
+    std::vector<cv::Mat> pafs(mapIdxOffset * 2);
+    for (size_t i = 0; i < pafs.size(); i++) {
+        pafs[i] = cv::Mat(dim2, dim3, CV_32FC1,
+                       predictions + i * dim2 * dim3);
+    }
+
+    auto hpe = new HPEOpenPose(0.1);
+    hpe->resizeFeatureMaps(heatMaps);
+    hpe->resizeFeatureMaps(pafs);
+
+    std::vector<HumanPose> poses = hpe->extractPoses(heatMaps, pafs);
+    std::cout << "pose: " << poses.size() << std::endl;
+
+    float upsampleRatio = (float) HPEOpenPose::upsampleRatio;
+    float scaleX = 8.0 / upsampleRatio * scale.x;
+    float scaleY = 8.0 / upsampleRatio * scale.y;
+
+    for (auto& pose : poses) {
+        for (auto& keypoint : pose.keypoints) {
+            if (keypoint != cv::Point2f(-1, -1)) {
+                keypoint.x *= scaleX;
+                keypoint.y *= scaleY;
+            }
+        }
+    }
+
+    elapsed = mach_absolute_time() - start;
+    double e = elapsed / 1000000000.0; // second
+    std::cout << "PCM/PAF time is " << e << " seconds." << std::endl;
+
+    delete(hpe);
+    vec.clear();
+    hvec.clear();
+    pvec.clear();
+
+    return renderHumanPose(poses, uiImage);
+}
+
+UIImage* renderHumanPose(std::vector<HumanPose> poses, UIImage* uiImage){
+
+    if (poses.size() < 1){
+        return uiImage;
+    }
+    cv::Mat outputImg;
+    UIImageToMat(uiImage, outputImg);
+    cv::cvtColor(outputImg, outputImg, cv::COLOR_RGB2RGBA);
+
+    static const cv::Scalar colors[HPEOpenPose::keypointsNumber] = {
+        cv::Scalar(255,  0, 85), cv::Scalar(255,  0,  0),
+        cv::Scalar(255, 85,  0), cv::Scalar(255,170,  0),
+        cv::Scalar(255,255,  0), cv::Scalar(170,255,  0),
+        cv::Scalar( 85,255,  0), cv::Scalar(  0,255,  0),
+        cv::Scalar(255,  0,  0), cv::Scalar(  0,255, 85),
+        cv::Scalar(  0,255,170), cv::Scalar(  0,255,255),
+        cv::Scalar(  0,170,255), cv::Scalar(  0, 85,255),
+        cv::Scalar(  0,  0,255), cv::Scalar(255,  0,170),
+        cv::Scalar(170,  0,255), cv::Scalar(255,  0,255),
+        cv::Scalar( 85,  0,255), cv::Scalar(0  ,  0,255),
+        cv::Scalar(  0,  0,255), cv::Scalar(0  ,  0,255),
+        cv::Scalar(  0,255,255), cv::Scalar(0  ,255,255),
+        cv::Scalar(  0,255,255)
+    };
+
+    static const std::pair<int, int> keypointsOP[] = {
+        {1, 8}, {1, 2}, {1, 5}, {2, 3}, {3, 4}, {5, 6}, {6, 7},
+        {8, 9}, {9, 10}, {10, 11}, {8, 12}, {12, 13}, {13, 14},
+        {1, 0}, {0, 15}, {15, 17}, {0, 16}, {16, 18},
+        {2, 17}, {5, 18}, {14, 19}, {19, 20},
+        {14, 21}, {11, 22}, {22, 23}, {11, 24}
+    };
+
+    const int stickWidth = 4;
+    const cv::Point2f absentKeypoint(-1.0f, -1.0f);
+    for (auto& pose : poses) {
+        for (size_t keypointIdx = 0; keypointIdx < pose.keypoints.size(); keypointIdx++) {
+            if (pose.keypoints[keypointIdx] != absentKeypoint) {
+                cv::circle(outputImg, pose.keypoints[keypointIdx], 4, colors[keypointIdx], -1);
+            }
+        }
+    }
+
+    std::vector<std::pair<int, int>> limbKeypointsIds;
+    if (!poses.empty()) {
+        limbKeypointsIds.insert(limbKeypointsIds.begin(), std::begin(keypointsOP), std::end(keypointsOP));
+    }
+
+    cv::Mat pane = outputImg.clone();
+    for (auto pose : poses) {
+        for (const auto& limbKeypointsId : limbKeypointsIds) {
+            if ((limbKeypointsId.first == 2 && limbKeypointsId.second == 17)
+                || (limbKeypointsId.first == 5 && limbKeypointsId.second == 18)){
+                continue;
+            }
+            std::pair<cv::Point2f, cv::Point2f> limbKeypoints(pose.keypoints[limbKeypointsId.first],
+                    pose.keypoints[limbKeypointsId.second]);
+            if (limbKeypoints.first == absentKeypoint
+                    || limbKeypoints.second == absentKeypoint) {
+                continue;
+            }
+
+            float meanX = (limbKeypoints.first.x + limbKeypoints.second.x) / 2;
+            float meanY = (limbKeypoints.first.y + limbKeypoints.second.y) / 2;
+            cv::Point difference = limbKeypoints.first - limbKeypoints.second;
+            double length = std::sqrt(difference.x * difference.x + difference.y * difference.y);
+            int angle = static_cast<int>(std::atan2(difference.y, difference.x) * 180 / CV_PI);
+            std::vector<cv::Point> polygon;
+            cv::ellipse2Poly(cv::Point2d(meanX, meanY), cv::Size2d(length / 2, stickWidth),
+                             angle, 0, 360, 1, polygon);
+            cv::fillConvexPoly(pane, polygon, colors[limbKeypointsId.second]);
+        }
+    }
+    cv::addWeighted(outputImg, 0.4, pane, 0.6, 0, outputImg);
+
+    UIImage *preview = MatToUIImage(outputImg);
+    outputImg.release();
+    return preview;
+}
+
+
+-(UIImage*) resizeImageExt: (UIImage*) uiImage
+                modelWidth: (int) modelWidth
+               modelHeight: (int) modelHeight
+                     scale: (CGPoint *) scale
+{
+    cv::Mat image;
+    UIImageToMat(uiImage, image);
+
+    cv::Rect roi;
+    auto paddedImage = resizeImageExt(image, modelWidth, modelHeight, RESIZE_KEEP_ASPECT, true, &roi);
+    std::cout << "roi: " << roi.width << "x" << roi.height << std::endl;
+
+    scale->x = (CGFloat)uiImage.size.width / (CGFloat)roi.width;
+    scale->y = (CGFloat)uiImage.size.height / (CGFloat)roi.height;
+    return MatToUIImage(paddedImage);
+}
+
+@end
+
+

PoseDecoder/Sources/PoseDecoder/include/PoseDecoder.h

@@ -0,0 +1,25 @@
+#import <Foundation/Foundation.h>
+#import <UIKit/UIKit.h>
+
+#ifndef PoseDecoder_h
+#define PoseDecoder_h
+
+@interface PoseDecoder : NSObject
+
+-(UIImage*) detection: (float *) data
+            dataSize: (int) dataSize
+            uiImage: (UIImage*) uiImage
+           modelWidth: (int) modelWidth
+          modelHeight: (int) modelHeight
+                scale: (CGPoint) scale
+;
+
+-(UIImage*) resizeImageExt: (UIImage*) uiImage
+                modelWidth: (int) modelWidth
+               modelHeight: (int) modelHeight
+                     scale: (CGPoint *) scale
+;
+
+@end
+
+#endif /* PoseDecoder_h */

PoseDecoder/Sources/PoseDecoderCPP/include/hpe_model_openpose.h

@@ -0,0 +1,43 @@
+/*
+// Copyright (C) 2018-2021 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+*/
+#pragma once
+#include "results.h"
+
+class HPEOpenPose {
+public:
+    /// Constructor
+    /// @param confidenceThreshold - threshold to eliminate low-confidence keypoints.
+    HPEOpenPose(float confidenceThreshold);
+
+    static const size_t keypointsNumber = 25;
+    static const int upsampleRatio = 4;
+
+    std::vector<HumanPose> extractPoses(const std::vector<cv::Mat>& heatMaps,
+                                        const std::vector<cv::Mat>& pafs) const;
+
+    void resizeFeatureMaps(std::vector<cv::Mat>& featureMaps) const;
+
+protected:
+
+    static const int minJointsNumber = 3;
+    static const int stride = 8;
+    static const cv::Vec3f meanPixel;
+    static const float minPeaksDistance;
+    static const float midPointsScoreThreshold;
+    static const float foundMidPointsRatioThreshold;
+    static const float minSubsetScore;
+    float confidenceThreshold;
+};

PoseDecoder/Sources/PoseDecoderCPP/include/openpose_decoder.h

@@ -0,0 +1,60 @@
+/*
+// Copyright (C) 2018-2021 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+*/
+#pragma once
+#include "opencv2/core.hpp"
+#include "results.h"
+
+struct Peak {
+    Peak(const int id = -1,
+         const cv::Point2f& pos = cv::Point2f(),
+         const float score = 0.0f);
+
+    int id;
+    cv::Point2f pos;
+    float score;
+};
+
+struct HumanPoseByPeaksIndices {
+    explicit HumanPoseByPeaksIndices(const int keypointsNumber);
+
+    std::vector<int> peaksIndices;
+    int nJoints;
+    float score;
+};
+
+struct TwoJointsConnection {
+    TwoJointsConnection(const int firstJointIdx,
+                        const int secondJointIdx,
+                        const float score);
+
+    int firstJointIdx;
+    int secondJointIdx;
+    float score;
+};
+
+void findPeaks(const std::vector<cv::Mat>& heatMaps,
+               const float minPeaksDistance,
+               std::vector<std::vector<Peak>>& allPeaks,
+               int heatMapId, float confidenceThreshold);
+
+std::vector<HumanPose> groupPeaksToPoses(
+        const std::vector<std::vector<Peak>>& allPeaks,
+        const std::vector<cv::Mat>& pafs,
+        const size_t keypointsNumber,
+        const float midPointsScoreThreshold,
+        const float foundMidPointsRatioThreshold,
+        const int minJointsNumber,
+        const float minSubsetScore);