From 430cad49a8bf454b4437dc8e3a091c8c96cfeaa8 Mon Sep 17 00:00:00 2001 From: EnoxSoftware Date: Mon, 24 Oct 2022 22:36:01 +0900 Subject: [PATCH] v1.0.5 Support for Hololens1 and Hololens2. (XR System: Legacy Built-in XR / XR Plugin Management WindowsMR / XR Plugin Management OpenXR) This closes #47, closes #45, closes #44, closes #43, closes #42, closes #36, closes #29 . --- ...sArUcoExample.meta => HLArUcoExample.meta} | 0 .../CameraParameters.cs | 124 +- .../CameraParameters.cs.meta | 0 .../HLArUcoCameraCalibrationExample.cs} | 2516 ++++++------- .../HLArUcoCameraCalibrationExample.cs.meta} | 0 .../HLArUcoCameraCalibrationExample.unity} | 428 ++- ...LArUcoCameraCalibrationExample.unity.meta} | 0 .../HLArUcoExample.cs} | 2035 +++++----- .../HLArUcoExample.cs.meta} | 0 .../HLArUcoExample.unity} | 259 +- .../HLArUcoExample.unity.meta} | 0 .../Materials.meta | 0 .../Materials/ArUcoMaterial.mat | 0 .../Materials/ArUcoMaterial.mat.meta | 0 ...er.meta => HLCameraIntrinsicsChecker.meta} | 0 .../HLCameraIntrinsicsChecker.unity} | 84 +- .../HLCameraIntrinsicsChecker.unity.meta} | 0 .../HLCameraIntrinsicsCheckerHelper.cs} | 636 ++-- .../HLCameraIntrinsicsCheckerHelper.cs.meta} | 0 ... => HLCameraStreamToMatHelperExample.meta} | 5 +- .../ComicFilter.cs | 350 +- .../ComicFilter.cs.meta | 0 .../HLCameraStreamToMatHelperExample.cs} | 788 ++-- .../HLCameraStreamToMatHelperExample.cs.meta} | 2 +- .../HLCameraStreamToMatHelperExample.unity} | 3276 +++++++++++------ ...CameraStreamToMatHelperExample.unity.meta} | 2 +- ...ample.meta => HLFaceDetectionExample.meta} | 0 .../HLFaceDetectionExample.cs} | 1740 ++++----- .../HLFaceDetectionExample.cs.meta} | 0 .../HLFaceDetectionExample.unity} | 220 +- .../HLFaceDetectionExample.unity.meta} | 0 ...xample.meta => HLPhotoCaptureExample.meta} | 0 .../HLPhotoCaptureExample.cs | 281 ++ .../HLPhotoCaptureExample.cs.meta} | 0 .../HLPhotoCaptureExample.unity} | 233 +- .../HLPhotoCaptureExample.unity.meta} | 0 .../HoloLensComicFilterExample.unity | 2799 -------------- .../HoloLensComicFilterExample.unity.meta | 6 - .../HoloLensFaceDetectionOverlayExample.meta | 9 - .../HoloLensFaceDetectionOverlayExample.cs | 846 ----- ...oloLensFaceDetectionOverlayExample.cs.meta | 12 - .../Materials.meta | 9 - .../Materials/AnonymousRectQuadMaterial.mat | 130 - .../AnonymousRectQuadMaterial.mat.meta | 8 - .../Materials/WierFrameShader.shader | 78 - .../Materials/WierFrameShader.shader.meta | 9 - .../Materials/WireFrameMaterial.mat | 175 - .../Materials/WireFrameMaterial.mat.meta | 8 - .../Overlay.meta | 9 - .../Overlay/AnonymousRectQuad.prefab | 108 - .../Overlay/AnonymousRectQuad.prefab.meta | 8 - .../Overlay/ObjectPool.cs | 139 - .../Overlay/ObjectPool.cs.meta | 12 - .../Overlay/RectOverlay.cs | 150 - .../Overlay/RectOverlay.cs.meta | 12 - .../Overlay/RectQuad.cs | 64 - .../Overlay/RectQuad.cs.meta | 12 - .../Overlay/RectQuad.prefab | 108 - .../Overlay/RectQuad.prefab.meta | 8 - .../Resources.meta | 9 - .../Resources/imageUnityNo02.png | Bin 57460 -> 0 bytes .../Resources/imageUnityNo02.png.meta | 84 - .../HoloLensPhotoCaptureExample.cs | 234 -- .../HoloLensWithOpenCVForUnityExample.cs | 221 +- .../HoloLensWithOpenCVForUnityExample.unity | 466 +-- ...material.mat => HolographicImageBlend.mat} | 7 +- ...at.meta => HolographicImageBlend.mat.meta} | 0 ...mat => HolographicImageBlendGrayscale.mat} | 7 +- ...> HolographicImageBlendGrayscale.mat.meta} | 0 ...end_material.mat => TextureSinglePass.mat} | 4 +- ...al.mat.meta => TextureSinglePass.mat.meta} | 0 ...ial.mat => TextureSinglePassGrayscale.mat} | 4 +- ...ta => TextureSinglePassGrayscale.mat.meta} | 0 .../Materials/grayscaleShader.shader | 41 - .../Materials/grayscaleShader.shader.meta | 9 - ...olographicImageBlendGrayscaleShader.shader | 94 - ...aphicImageBlendGrayscaleShader.shader.meta | 9 - .../holographicImageBlendShader.shader | 92 - .../holographicImageBlendShader.shader.meta | 9 - .../Scripts/RectangleTracker.cs | 1044 +++--- .../RectangleTracker/Scripts/TrackedObject.cs | 102 +- .../RectangleTracker/Scripts/TrackedRect.cs | 40 +- .../Scripts/TrackerParameters.cs | 66 +- .../Scripts/ARGameObject.cs | 164 +- .../Scripts/Utils/DebugUtils.cs | 180 +- ...Helper.cs => HLCameraStreamToMatHelper.cs} | 2033 +++++----- ...meta => HLCameraStreamToMatHelper.cs.meta} | 0 .../ShowLicense.cs | 58 +- BuildSettings.jpg | Bin 176342 -> 302432 bytes MRTKProjectConfigurator.jpg | Bin 119264 -> 0 bytes PlayerSettings.jpg | Bin 522115 -> 294470 bytes ProjectAssets.jpg | Bin 223999 -> 84423 bytes README.md | 54 +- 93 files changed, 9832 insertions(+), 12897 deletions(-) rename Assets/HoloLensWithOpenCVForUnityExample/{HoloLensArUcoExample.meta => HLArUcoExample.meta} (100%) rename Assets/HoloLensWithOpenCVForUnityExample/{HoloLensArUcoExample => HLArUcoExample}/CameraParameters.cs (96%) rename Assets/HoloLensWithOpenCVForUnityExample/{HoloLensArUcoExample => HLArUcoExample}/CameraParameters.cs.meta (100%) rename Assets/HoloLensWithOpenCVForUnityExample/{HoloLensArUcoExample/HoloLensArUcoCameraCalibrationExample.cs => HLArUcoExample/HLArUcoCameraCalibrationExample.cs} (96%) rename Assets/HoloLensWithOpenCVForUnityExample/{HoloLensArUcoExample/HoloLensArUcoCameraCalibrationExample.cs.meta => HLArUcoExample/HLArUcoCameraCalibrationExample.cs.meta} (100%) rename Assets/HoloLensWithOpenCVForUnityExample/{HoloLensArUcoExample/HoloLensArUcoCameraCalibrationExample.unity => HLArUcoExample/HLArUcoCameraCalibrationExample.unity} (93%) rename Assets/HoloLensWithOpenCVForUnityExample/{HoloLensArUcoExample/HoloLensArUcoCameraCalibrationExample.unity.meta => HLArUcoExample/HLArUcoCameraCalibrationExample.unity.meta} (100%) rename Assets/HoloLensWithOpenCVForUnityExample/{HoloLensArUcoExample/HoloLensArUcoExample.cs => HLArUcoExample/HLArUcoExample.cs} (94%) rename Assets/HoloLensWithOpenCVForUnityExample/{HoloLensArUcoExample/HoloLensArUcoExample.cs.meta => HLArUcoExample/HLArUcoExample.cs.meta} (100%) rename Assets/HoloLensWithOpenCVForUnityExample/{HoloLensArUcoExample/HoloLensArUcoExample.unity => HLArUcoExample/HLArUcoExample.unity} (93%) rename Assets/HoloLensWithOpenCVForUnityExample/{HoloLensArUcoExample/HoloLensArUcoExample.unity.meta => HLArUcoExample/HLArUcoExample.unity.meta} (100%) rename Assets/HoloLensWithOpenCVForUnityExample/{HoloLensArUcoExample => HLArUcoExample}/Materials.meta (100%) rename Assets/HoloLensWithOpenCVForUnityExample/{HoloLensArUcoExample => HLArUcoExample}/Materials/ArUcoMaterial.mat (100%) rename Assets/HoloLensWithOpenCVForUnityExample/{HoloLensArUcoExample => HLArUcoExample}/Materials/ArUcoMaterial.mat.meta (100%) rename Assets/HoloLensWithOpenCVForUnityExample/{HoloLensCameraIntrinsicsChecker.meta => HLCameraIntrinsicsChecker.meta} (100%) rename Assets/HoloLensWithOpenCVForUnityExample/{HoloLensCameraIntrinsicsChecker/HoloLensCameraIntrinsicsChecker.unity => HLCameraIntrinsicsChecker/HLCameraIntrinsicsChecker.unity} (93%) rename Assets/HoloLensWithOpenCVForUnityExample/{HoloLensCameraIntrinsicsChecker/HoloLensCameraIntrinsicsChecker.unity.meta => HLCameraIntrinsicsChecker/HLCameraIntrinsicsChecker.unity.meta} (100%) rename Assets/HoloLensWithOpenCVForUnityExample/{HoloLensCameraIntrinsicsChecker/CameraIntrinsicsCheckerHelper.cs => HLCameraIntrinsicsChecker/HLCameraIntrinsicsCheckerHelper.cs} (64%) rename Assets/HoloLensWithOpenCVForUnityExample/{HoloLensCameraIntrinsicsChecker/CameraIntrinsicsCheckerHelper.cs.meta => HLCameraIntrinsicsChecker/HLCameraIntrinsicsCheckerHelper.cs.meta} (100%) rename Assets/HoloLensWithOpenCVForUnityExample/{HoloLensComicFilterExample.meta => HLCameraStreamToMatHelperExample.meta} (57%) rename Assets/HoloLensWithOpenCVForUnityExample/{HoloLensComicFilterExample => HLCameraStreamToMatHelperExample}/ComicFilter.cs (96%) rename Assets/HoloLensWithOpenCVForUnityExample/{HoloLensComicFilterExample => HLCameraStreamToMatHelperExample}/ComicFilter.cs.meta (100%) rename Assets/HoloLensWithOpenCVForUnityExample/{HoloLensComicFilterExample/HoloLensComicFilterExample.cs => HLCameraStreamToMatHelperExample/HLCameraStreamToMatHelperExample.cs} (68%) rename Assets/HoloLensWithOpenCVForUnityExample/{HoloLensComicFilterExample/HoloLensComicFilterExample.cs.meta => HLCameraStreamToMatHelperExample/HLCameraStreamToMatHelperExample.cs.meta} (82%) rename Assets/HoloLensWithOpenCVForUnityExample/{HoloLensFaceDetectionOverlayExample/HoloLensFaceDetectionOverlayExample.unity => HLCameraStreamToMatHelperExample/HLCameraStreamToMatHelperExample.unity} (63%) rename Assets/HoloLensWithOpenCVForUnityExample/{HoloLensFaceDetectionOverlayExample/HoloLensFaceDetectionOverlayExample.unity.meta => HLCameraStreamToMatHelperExample/HLCameraStreamToMatHelperExample.unity.meta} (70%) rename Assets/HoloLensWithOpenCVForUnityExample/{HoloLensFaceDetectionExample.meta => HLFaceDetectionExample.meta} (100%) rename Assets/HoloLensWithOpenCVForUnityExample/{HoloLensFaceDetectionExample/HoloLensFaceDetectionExample.cs => HLFaceDetectionExample/HLFaceDetectionExample.cs} (91%) rename Assets/HoloLensWithOpenCVForUnityExample/{HoloLensFaceDetectionExample/HoloLensFaceDetectionExample.cs.meta => HLFaceDetectionExample/HLFaceDetectionExample.cs.meta} (100%) rename Assets/HoloLensWithOpenCVForUnityExample/{HoloLensFaceDetectionExample/HoloLensFaceDetectionExample.unity => HLFaceDetectionExample/HLFaceDetectionExample.unity} (93%) rename Assets/HoloLensWithOpenCVForUnityExample/{HoloLensFaceDetectionExample/HoloLensFaceDetectionExample.unity.meta => HLFaceDetectionExample/HLFaceDetectionExample.unity.meta} (100%) rename Assets/HoloLensWithOpenCVForUnityExample/{HoloLensPhotoCaptureExample.meta => HLPhotoCaptureExample.meta} (100%) create mode 100644 Assets/HoloLensWithOpenCVForUnityExample/HLPhotoCaptureExample/HLPhotoCaptureExample.cs rename Assets/HoloLensWithOpenCVForUnityExample/{HoloLensPhotoCaptureExample/HoloLensPhotoCaptureExample.cs.meta => HLPhotoCaptureExample/HLPhotoCaptureExample.cs.meta} (100%) rename Assets/HoloLensWithOpenCVForUnityExample/{HoloLensPhotoCaptureExample/HoloLensPhotoCaptureExample.unity => HLPhotoCaptureExample/HLPhotoCaptureExample.unity} (89%) rename Assets/HoloLensWithOpenCVForUnityExample/{HoloLensPhotoCaptureExample/HoloLensPhotoCaptureExample.unity.meta => HLPhotoCaptureExample/HLPhotoCaptureExample.unity.meta} (100%) delete mode 100644 Assets/HoloLensWithOpenCVForUnityExample/HoloLensComicFilterExample/HoloLensComicFilterExample.unity delete mode 100644 Assets/HoloLensWithOpenCVForUnityExample/HoloLensComicFilterExample/HoloLensComicFilterExample.unity.meta delete mode 100644 Assets/HoloLensWithOpenCVForUnityExample/HoloLensFaceDetectionOverlayExample.meta delete mode 100644 Assets/HoloLensWithOpenCVForUnityExample/HoloLensFaceDetectionOverlayExample/HoloLensFaceDetectionOverlayExample.cs delete mode 100644 Assets/HoloLensWithOpenCVForUnityExample/HoloLensFaceDetectionOverlayExample/HoloLensFaceDetectionOverlayExample.cs.meta delete mode 100644 Assets/HoloLensWithOpenCVForUnityExample/HoloLensFaceDetectionOverlayExample/Materials.meta delete mode 100644 Assets/HoloLensWithOpenCVForUnityExample/HoloLensFaceDetectionOverlayExample/Materials/AnonymousRectQuadMaterial.mat delete mode 100644 Assets/HoloLensWithOpenCVForUnityExample/HoloLensFaceDetectionOverlayExample/Materials/AnonymousRectQuadMaterial.mat.meta delete mode 100644 Assets/HoloLensWithOpenCVForUnityExample/HoloLensFaceDetectionOverlayExample/Materials/WierFrameShader.shader delete mode 100644 Assets/HoloLensWithOpenCVForUnityExample/HoloLensFaceDetectionOverlayExample/Materials/WierFrameShader.shader.meta delete mode 100644 Assets/HoloLensWithOpenCVForUnityExample/HoloLensFaceDetectionOverlayExample/Materials/WireFrameMaterial.mat delete mode 100644 Assets/HoloLensWithOpenCVForUnityExample/HoloLensFaceDetectionOverlayExample/Materials/WireFrameMaterial.mat.meta delete mode 100644 Assets/HoloLensWithOpenCVForUnityExample/HoloLensFaceDetectionOverlayExample/Overlay.meta delete mode 100644 Assets/HoloLensWithOpenCVForUnityExample/HoloLensFaceDetectionOverlayExample/Overlay/AnonymousRectQuad.prefab delete mode 100644 Assets/HoloLensWithOpenCVForUnityExample/HoloLensFaceDetectionOverlayExample/Overlay/AnonymousRectQuad.prefab.meta delete mode 100644 Assets/HoloLensWithOpenCVForUnityExample/HoloLensFaceDetectionOverlayExample/Overlay/ObjectPool.cs delete mode 100644 Assets/HoloLensWithOpenCVForUnityExample/HoloLensFaceDetectionOverlayExample/Overlay/ObjectPool.cs.meta delete mode 100644 Assets/HoloLensWithOpenCVForUnityExample/HoloLensFaceDetectionOverlayExample/Overlay/RectOverlay.cs delete mode 100644 Assets/HoloLensWithOpenCVForUnityExample/HoloLensFaceDetectionOverlayExample/Overlay/RectOverlay.cs.meta delete mode 100644 Assets/HoloLensWithOpenCVForUnityExample/HoloLensFaceDetectionOverlayExample/Overlay/RectQuad.cs delete mode 100644 Assets/HoloLensWithOpenCVForUnityExample/HoloLensFaceDetectionOverlayExample/Overlay/RectQuad.cs.meta delete mode 100644 Assets/HoloLensWithOpenCVForUnityExample/HoloLensFaceDetectionOverlayExample/Overlay/RectQuad.prefab delete mode 100644 Assets/HoloLensWithOpenCVForUnityExample/HoloLensFaceDetectionOverlayExample/Overlay/RectQuad.prefab.meta delete mode 100644 Assets/HoloLensWithOpenCVForUnityExample/HoloLensFaceDetectionOverlayExample/Resources.meta delete mode 100644 Assets/HoloLensWithOpenCVForUnityExample/HoloLensFaceDetectionOverlayExample/Resources/imageUnityNo02.png delete mode 100644 Assets/HoloLensWithOpenCVForUnityExample/HoloLensFaceDetectionOverlayExample/Resources/imageUnityNo02.png.meta delete mode 100644 Assets/HoloLensWithOpenCVForUnityExample/HoloLensPhotoCaptureExample/HoloLensPhotoCaptureExample.cs rename Assets/HoloLensWithOpenCVForUnityExample/Materials/{holographicImageBlendGrayscale_material.mat => HolographicImageBlend.mat} (81%) rename Assets/HoloLensWithOpenCVForUnityExample/Materials/{holographicImageBlend_material.mat.meta => HolographicImageBlend.mat.meta} (100%) rename Assets/HoloLensWithOpenCVForUnityExample/Materials/{quad_material.mat => HolographicImageBlendGrayscale.mat} (78%) rename Assets/HoloLensWithOpenCVForUnityExample/Materials/{holographicImageBlendGrayscale_material.mat.meta => HolographicImageBlendGrayscale.mat.meta} (100%) rename Assets/HoloLensWithOpenCVForUnityExample/Materials/{holographicImageBlend_material.mat => TextureSinglePass.mat} (85%) rename Assets/HoloLensWithOpenCVForUnityExample/Materials/{quad_material.mat.meta => TextureSinglePass.mat.meta} (100%) rename Assets/HoloLensWithOpenCVForUnityExample/Materials/{quadGrayscale_material.mat => TextureSinglePassGrayscale.mat} (86%) rename Assets/HoloLensWithOpenCVForUnityExample/Materials/{quadGrayscale_material.mat.meta => TextureSinglePassGrayscale.mat.meta} (100%) delete mode 100644 Assets/HoloLensWithOpenCVForUnityExample/Materials/grayscaleShader.shader delete mode 100644 Assets/HoloLensWithOpenCVForUnityExample/Materials/grayscaleShader.shader.meta delete mode 100644 Assets/HoloLensWithOpenCVForUnityExample/Materials/holographicImageBlendGrayscaleShader.shader delete mode 100644 Assets/HoloLensWithOpenCVForUnityExample/Materials/holographicImageBlendGrayscaleShader.shader.meta delete mode 100644 Assets/HoloLensWithOpenCVForUnityExample/Materials/holographicImageBlendShader.shader delete mode 100644 Assets/HoloLensWithOpenCVForUnityExample/Materials/holographicImageBlendShader.shader.meta rename Assets/HoloLensWithOpenCVForUnityExample/Scripts/Utils/{HololensCameraStreamToMatHelper.cs => HLCameraStreamToMatHelper.cs} (76%) rename Assets/HoloLensWithOpenCVForUnityExample/Scripts/Utils/{HololensCameraStreamToMatHelper.cs.meta => HLCameraStreamToMatHelper.cs.meta} (100%) delete mode 100644 MRTKProjectConfigurator.jpg diff --git a/Assets/HoloLensWithOpenCVForUnityExample/HoloLensArUcoExample.meta b/Assets/HoloLensWithOpenCVForUnityExample/HLArUcoExample.meta similarity index 100% rename from Assets/HoloLensWithOpenCVForUnityExample/HoloLensArUcoExample.meta rename to Assets/HoloLensWithOpenCVForUnityExample/HLArUcoExample.meta diff --git a/Assets/HoloLensWithOpenCVForUnityExample/HoloLensArUcoExample/CameraParameters.cs b/Assets/HoloLensWithOpenCVForUnityExample/HLArUcoExample/CameraParameters.cs similarity index 96% rename from Assets/HoloLensWithOpenCVForUnityExample/HoloLensArUcoExample/CameraParameters.cs rename to Assets/HoloLensWithOpenCVForUnityExample/HLArUcoExample/CameraParameters.cs index 4efe299..f0b69c2 100644 --- a/Assets/HoloLensWithOpenCVForUnityExample/HoloLensArUcoExample/CameraParameters.cs +++ b/Assets/HoloLensWithOpenCVForUnityExample/HLArUcoExample/CameraParameters.cs @@ -1,62 +1,62 @@ -using System; -using OpenCVForUnity.CoreModule; - -namespace HoloLensWithOpenCVForUnityExample -{ - [System.Serializable] - public struct CameraParameters - { - public string calibration_date; - public int frames_count; - public int image_width; - public int image_height; - public int calibration_flags; - public double[] camera_matrix; - public double[] distortion_coefficients; - public double avg_reprojection_error; - - public CameraParameters(int frames_count, int image_width, int image_height, int calibration_flags, double[] camera_matrix, double[] distortion_coefficients, double avg_reprojection_error) - { - this.calibration_date = DateTime.Now.ToString(); - this.frames_count = frames_count; - this.image_width = image_width; - this.image_height = image_height; - this.calibration_flags = calibration_flags; - this.camera_matrix = camera_matrix; - this.distortion_coefficients = distortion_coefficients; - this.avg_reprojection_error = avg_reprojection_error; - } - - public CameraParameters(int frames_count, int image_width, int image_height, int calibration_flags, Mat camera_matrix, Mat distortion_coefficients, double avg_reprojection_error) - { - double[] camera_matrixArr = new double[camera_matrix.total()]; - camera_matrix.get(0, 0, camera_matrixArr); - - double[] distortion_coefficientsArr = new double[distortion_coefficients.total()]; - distortion_coefficients.get(0, 0, distortion_coefficientsArr); - - this.calibration_date = DateTime.Now.ToString(); - this.frames_count = frames_count; - this.image_width = image_width; - this.image_height = image_height; - this.calibration_flags = calibration_flags; - this.camera_matrix = camera_matrixArr; - this.distortion_coefficients = distortion_coefficientsArr; - this.avg_reprojection_error = avg_reprojection_error; - } - - public Mat GetCameraMatrix() - { - Mat m = new Mat(3, 3, CvType.CV_64FC1); - m.put(0, 0, camera_matrix); - return m; - } - - public Mat GetDistortionCoefficients() - { - Mat m = new Mat(distortion_coefficients.Length, 1, CvType.CV_64FC1); - m.put(0, 0, distortion_coefficients); - return m; - } - } -} +using System; +using OpenCVForUnity.CoreModule; + +namespace HoloLensWithOpenCVForUnityExample +{ + [System.Serializable] + public struct CameraParameters + { + public string calibration_date; + public int frames_count; + public int image_width; + public int image_height; + public int calibration_flags; + public double[] camera_matrix; + public double[] distortion_coefficients; + public double avg_reprojection_error; + + public CameraParameters(int frames_count, int image_width, int image_height, int calibration_flags, double[] camera_matrix, double[] distortion_coefficients, double avg_reprojection_error) + { + this.calibration_date = DateTime.Now.ToString(); + this.frames_count = frames_count; + this.image_width = image_width; + this.image_height = image_height; + this.calibration_flags = calibration_flags; + this.camera_matrix = camera_matrix; + this.distortion_coefficients = distortion_coefficients; + this.avg_reprojection_error = avg_reprojection_error; + } + + public CameraParameters(int frames_count, int image_width, int image_height, int calibration_flags, Mat camera_matrix, Mat distortion_coefficients, double avg_reprojection_error) + { + double[] camera_matrixArr = new double[camera_matrix.total()]; + camera_matrix.get(0, 0, camera_matrixArr); + + double[] distortion_coefficientsArr = new double[distortion_coefficients.total()]; + distortion_coefficients.get(0, 0, distortion_coefficientsArr); + + this.calibration_date = DateTime.Now.ToString(); + this.frames_count = frames_count; + this.image_width = image_width; + this.image_height = image_height; + this.calibration_flags = calibration_flags; + this.camera_matrix = camera_matrixArr; + this.distortion_coefficients = distortion_coefficientsArr; + this.avg_reprojection_error = avg_reprojection_error; + } + + public Mat GetCameraMatrix() + { + Mat m = new Mat(3, 3, CvType.CV_64FC1); + m.put(0, 0, camera_matrix); + return m; + } + + public Mat GetDistortionCoefficients() + { + Mat m = new Mat(distortion_coefficients.Length, 1, CvType.CV_64FC1); + m.put(0, 0, distortion_coefficients); + return m; + } + } +} diff --git a/Assets/HoloLensWithOpenCVForUnityExample/HoloLensArUcoExample/CameraParameters.cs.meta b/Assets/HoloLensWithOpenCVForUnityExample/HLArUcoExample/CameraParameters.cs.meta similarity index 100% rename from Assets/HoloLensWithOpenCVForUnityExample/HoloLensArUcoExample/CameraParameters.cs.meta rename to Assets/HoloLensWithOpenCVForUnityExample/HLArUcoExample/CameraParameters.cs.meta diff --git a/Assets/HoloLensWithOpenCVForUnityExample/HoloLensArUcoExample/HoloLensArUcoCameraCalibrationExample.cs b/Assets/HoloLensWithOpenCVForUnityExample/HLArUcoExample/HLArUcoCameraCalibrationExample.cs similarity index 96% rename from Assets/HoloLensWithOpenCVForUnityExample/HoloLensArUcoExample/HoloLensArUcoCameraCalibrationExample.cs rename to Assets/HoloLensWithOpenCVForUnityExample/HLArUcoExample/HLArUcoCameraCalibrationExample.cs index 1359ce8..388dc9b 100644 --- a/Assets/HoloLensWithOpenCVForUnityExample/HoloLensArUcoExample/HoloLensArUcoCameraCalibrationExample.cs +++ b/Assets/HoloLensWithOpenCVForUnityExample/HLArUcoExample/HLArUcoCameraCalibrationExample.cs @@ -1,1256 +1,1262 @@ -using UnityEngine; -using UnityEngine.UI; -using UnityEngine.SceneManagement; -using System; -using System.Collections; -using System.Collections.Generic; -using System.Xml.Serialization; -using System.IO; -using System.Linq; -using OpenCVForUnity.CoreModule; -using OpenCVForUnity.ArucoModule; -using OpenCVForUnity.UnityUtils.Helper; -using OpenCVForUnity.ImgprocModule; -using OpenCVForUnity.UnityUtils; -using OpenCVForUnity.Calib3dModule; -using OpenCVForUnity.ImgcodecsModule; -using HoloLensWithOpenCVForUnity.UnityUtils.Helper; -using Microsoft.MixedReality.Toolkit.Input; - -namespace HoloLensWithOpenCVForUnityExample -{ - /// - /// HoloLens ArUco Camera Calibration Example - /// An example of camera calibration using the aruco module. - /// Referring to https://github.com/opencv/opencv_contrib/blob/master/modules/aruco/samples/calibrate_camera.cpp. - /// https://github.com/opencv/opencv/blob/master/samples/cpp/tutorial_code/calib3d/camera_calibration/camera_calibration.cpp - /// https://docs.opencv.org/3.2.0/da/d13/tutorial_aruco_calibration.html - /// https://docs.opencv.org/3.4.0/d7/d21/tutorial_interactive_calibration.html - /// - [RequireComponent(typeof(HololensCameraStreamToMatHelper))] - public class HoloLensArUcoCameraCalibrationExample : MonoBehaviour - { - /// - /// The preview quad. - /// - public GameObject previewQuad; - - /// - /// The marker type. - /// - public MarkerType markerType = MarkerType.ChArUcoBoard; - - /// - /// The marker type dropdown. - /// - public Dropdown markerTypeDropdown; - - /// - /// The dictionary identifier. - /// - public ArUcoDictionary dictionaryId = ArUcoDictionary.DICT_6X6_250; - - /// - /// The dictionary id dropdown. - /// - public Dropdown dictionaryIdDropdown; - - /// - /// Number of squares in X direction. - /// - public NumberOfSquaresX squaresX = NumberOfSquaresX.X_5; - - /// - /// The squares X dropdown. - /// - public Dropdown squaresXDropdown; - - /// - /// Number of squares in X direction. - /// - public NumberOfSquaresY squaresY = NumberOfSquaresY.Y_7; - - /// - /// The squares X dropdown. - /// - public Dropdown squaresYDropdown; - - /// - /// The save path input field. - /// - public InputField savePathInputField; - - /// - /// Determines if refine marker detection. (only valid for ArUco boards) - /// - public bool refineMarkerDetection = true; - - - [Header("Extra Option")] - - /// - /// Determines if calibrates camera using the list of calibration images. - /// - [TooltipAttribute("Determines if calibrates camera using the list of calibration images.")] - public bool isImagesInputMode = false; - - /// - /// The calibration images directory path. - /// Set a relative directory path from the starting point of the "StreamingAssets" folder. e.g. "calibration_images/". - /// - [TooltipAttribute("Set a relative directory path from the starting point of the \"StreamingAssets\" folder. e.g. \"calibration_images\"")] - public string calibrationImagesDirectory = "calibration_images"; - - /// - /// The texture. - /// - Texture2D texture; - - /// - /// The webcam texture to mat helper. - /// - HololensCameraStreamToMatHelper webCamTextureToMatHelper; - - /// - /// The bgr mat. - /// - Mat bgrMat; - - /// - /// The rgba mat. - /// - Mat rgbaMat; - - /// - /// The cameraparam matrix. - /// - Mat camMatrix; - - /// - /// The distortion coeffs. - /// - MatOfDouble distCoeffs; - - /// - /// The identifiers. - /// - Mat ids; - - /// - /// The corners. - /// - List corners; - - /// - /// The rejected corners. - /// - List rejectedCorners; - - /// - /// The rvecs. - /// - List rvecs; - - /// - /// The tvecs. - /// - List tvecs; - - /// - /// The detector parameters. - /// - DetectorParameters detectorParams; - - /// - /// The dictionary. - /// - Dictionary dictionary; - - /// - /// The recovered identifiers. - /// - Mat recoveredIdxs; - - const int calibrationFlags = 0;// Calib3d.CALIB_FIX_K3 | Calib3d.CALIB_FIX_K4 | Calib3d.CALIB_FIX_K5 - double repErr = 0; - bool shouldCaptureFrame = false; - - // for ChArUcoBoard. - // chessboard square side length (normally in meters) - const float chArUcoBoradSquareLength = 0.04f; - // marker side length (same unit than squareLength) - const float chArUcoBoradMarkerLength = 0.02f; - const int charucoMinMarkers = 2; - Mat charucoCorners; - Mat charucoIds; - CharucoBoard charucoBoard; - List> allCorners; - List allIds; - List allImgs; - - // for OthearMarkers. - // square size in some user-defined units (1 by default) - const float squareSize = 1f; - List imagePoints; - bool isInitialized = false; - bool isCalibrating = false; - - // Use this for initialization - IEnumerator Start() - { - webCamTextureToMatHelper = gameObject.GetComponent(); - webCamTextureToMatHelper.outputColorFormat = WebCamTextureToMatHelper.ColorFormat.GRAY; - - // fix the screen orientation. - Screen.orientation = ScreenOrientation.LandscapeLeft; - // wait for the screen orientation to change. - yield return null; - - if (markerTypeDropdown.value != (int)markerType || dictionaryIdDropdown.value != (int)dictionaryId - || squaresXDropdown.value != (int)squaresX - 1 || squaresYDropdown.value != (int)squaresY - 1) - { - markerTypeDropdown.value = (int)markerType; - dictionaryIdDropdown.value = (int)dictionaryId; - squaresXDropdown.value = (int)squaresX - 1; - squaresYDropdown.value = (int)squaresY - 1; - } - dictionaryIdDropdown.interactable = (markerType == MarkerType.ChArUcoBoard); - -#if UNITY_WEBGL && !UNITY_EDITOR - isImagesInputMode = false; -#endif - if (isImagesInputMode) - { - isImagesInputMode = InitializeImagesInputMode(); - } - - if (!isImagesInputMode) - { - webCamTextureToMatHelper.Initialize(); - } - } - - /// - /// Raises the webcam texture to mat helper initialized event. - /// - public void OnWebCamTextureToMatHelperInitialized() - { - Debug.Log("OnWebCamTextureToMatHelperInitialized"); - - Mat webCamTextureMat = webCamTextureToMatHelper.GetMat(); - - InitializeCalibraton(webCamTextureMat); - - // if WebCamera is frontFaceing, flip Mat. - webCamTextureToMatHelper.flipHorizontal = webCamTextureToMatHelper.IsFrontFacing(); - } - - /// - /// Raises the webcam texture to mat helper disposed event. - /// - public void OnWebCamTextureToMatHelperDisposed() - { - Debug.Log("OnWebCamTextureToMatHelperDisposed"); - - DisposeCalibraton(); - } - - /// - /// Raises the webcam texture to mat helper error occurred event. - /// - /// Error code. - public void OnWebCamTextureToMatHelperErrorOccurred(WebCamTextureToMatHelper.ErrorCode errorCode) - { - Debug.Log("OnWebCamTextureToMatHelperErrorOccurred " + errorCode); - } - - // Update is called once per frame - void Update() - { - if (isImagesInputMode) - return; - - if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) - { - - Mat grayMat = webCamTextureToMatHelper.GetMat(); - - if (shouldCaptureFrame) - { - shouldCaptureFrame = false; - Mat frameMat = grayMat.clone(); - double e = CaptureFrame(frameMat); - if (e > 0) - repErr = e; - } - - DrawFrame(grayMat, bgrMat); - Imgproc.cvtColor(bgrMat, rgbaMat, Imgproc.COLOR_BGR2RGBA); - - Utils.fastMatToTexture2D(rgbaMat, texture); - } - } - - private void InitializeCalibraton(Mat frameMat) - { - Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation); - - float width = frameMat.width(); - float height = frameMat.height(); - - texture = new Texture2D(frameMat.cols(), frameMat.rows(), TextureFormat.RGBA32, false); - texture.wrapMode = TextureWrapMode.Clamp; - - previewQuad.GetComponent().material.mainTexture = texture; - previewQuad.transform.localScale = new Vector3(0.2f * width / height, 0.2f, 1); - - float imageSizeScale = 1.0f; - - // set cameraparam. - camMatrix = CreateCameraMatrix(width, height); - Debug.Log("camMatrix " + camMatrix.dump()); - - distCoeffs = new MatOfDouble(0, 0, 0, 0, 0); - Debug.Log("distCoeffs " + distCoeffs.dump()); - - // calibration camera. - Size imageSize = new Size(width * imageSizeScale, height * imageSizeScale); - double apertureWidth = 0; - double apertureHeight = 0; - double[] fovx = new double[1]; - double[] fovy = new double[1]; - double[] focalLength = new double[1]; - Point principalPoint = new Point(0, 0); - double[] aspectratio = new double[1]; - - Calib3d.calibrationMatrixValues(camMatrix, imageSize, apertureWidth, apertureHeight, fovx, fovy, focalLength, principalPoint, aspectratio); - - Debug.Log("imageSize " + imageSize.ToString()); - Debug.Log("apertureWidth " + apertureWidth); - Debug.Log("apertureHeight " + apertureHeight); - Debug.Log("fovx " + fovx[0]); - Debug.Log("fovy " + fovy[0]); - Debug.Log("focalLength " + focalLength[0]); - Debug.Log("principalPoint " + principalPoint.ToString()); - Debug.Log("aspectratio " + aspectratio[0]); - - - bgrMat = new Mat(frameMat.rows(), frameMat.cols(), CvType.CV_8UC3); - rgbaMat = new Mat(frameMat.rows(), frameMat.cols(), CvType.CV_8UC4); - ids = new Mat(); - corners = new List(); - rejectedCorners = new List(); - rvecs = new List(); - tvecs = new List(); - - detectorParams = DetectorParameters.create(); - detectorParams.set_cornerRefinementMethod(1);// do cornerSubPix() of OpenCV. - dictionary = Aruco.getPredefinedDictionary((int)dictionaryId); - - recoveredIdxs = new Mat(); - - charucoCorners = new Mat(); - charucoIds = new Mat(); - charucoBoard = CharucoBoard.create((int)squaresX, (int)squaresY, chArUcoBoradSquareLength, chArUcoBoradMarkerLength, dictionary); - - - allCorners = new List>(); - allIds = new List(); - allImgs = new List(); - - imagePoints = new List(); - - isInitialized = true; - } - - private void DisposeCalibraton() - { - ResetCalibration(); - - if (bgrMat != null) - bgrMat.Dispose(); - if (rgbaMat != null) - rgbaMat.Dispose(); - if (ids != null) - ids.Dispose(); - foreach (var item in corners) - { - item.Dispose(); - } - corners.Clear(); - foreach (var item in rejectedCorners) - { - item.Dispose(); - } - rejectedCorners.Clear(); - foreach (var item in rvecs) - { - item.Dispose(); - } - rvecs.Clear(); - foreach (var item in tvecs) - { - item.Dispose(); - } - tvecs.Clear(); - - if (recoveredIdxs != null) - recoveredIdxs.Dispose(); - - if (charucoCorners != null) - charucoCorners.Dispose(); - if (charucoIds != null) - charucoIds.Dispose(); - if (charucoBoard != null) - charucoBoard.Dispose(); - - isInitialized = false; - } - - private void DrawFrame(Mat grayMat, Mat bgrMat) - { - Imgproc.cvtColor(grayMat, bgrMat, Imgproc.COLOR_GRAY2BGR); - - switch (markerType) - { - default: - case MarkerType.ChArUcoBoard: - // detect markers. - Aruco.detectMarkers(grayMat, dictionary, corners, ids, detectorParams, rejectedCorners, camMatrix, distCoeffs); - - // refine marker detection. - if (refineMarkerDetection) - { - Aruco.refineDetectedMarkers(grayMat, charucoBoard, corners, ids, rejectedCorners, camMatrix, distCoeffs, 10f, 3f, true, recoveredIdxs, detectorParams); - } - - // if at least one marker detected - if (ids.total() > 0) - { - Aruco.interpolateCornersCharuco(corners, ids, grayMat, charucoBoard, charucoCorners, charucoIds, camMatrix, distCoeffs, charucoMinMarkers); - - // draw markers. - Aruco.drawDetectedMarkers(bgrMat, corners, ids, new Scalar(0, 255, 0, 255)); - // if at least one charuco corner detected - if (charucoIds.total() > 0) - { - Aruco.drawDetectedCornersCharuco(bgrMat, charucoCorners, charucoIds, new Scalar(0, 0, 255, 255)); - } - } - break; - case MarkerType.ChessBoard: - case MarkerType.CirclesGlid: - case MarkerType.AsymmetricCirclesGlid: - // detect markers. - MatOfPoint2f points = new MatOfPoint2f(); - bool found = false; - - switch (markerType) - { - default: - case MarkerType.ChessBoard: - found = Calib3d.findChessboardCorners(grayMat, new Size((int)squaresX, (int)squaresY), points, Calib3d.CALIB_CB_ADAPTIVE_THRESH | Calib3d.CALIB_CB_FAST_CHECK | Calib3d.CALIB_CB_NORMALIZE_IMAGE); - break; - case MarkerType.CirclesGlid: - found = Calib3d.findCirclesGrid(grayMat, new Size((int)squaresX, (int)squaresY), points, Calib3d.CALIB_CB_SYMMETRIC_GRID); - break; - case MarkerType.AsymmetricCirclesGlid: - found = Calib3d.findCirclesGrid(grayMat, new Size((int)squaresX, (int)squaresY), points, Calib3d.CALIB_CB_ASYMMETRIC_GRID); - break; - } - - if (found) - { - if (markerType == MarkerType.ChessBoard) - Imgproc.cornerSubPix(grayMat, points, new Size(5, 5), new Size(-1, -1), new TermCriteria(TermCriteria.EPS + TermCriteria.COUNT, 30, 0.1)); - - // draw markers. - Calib3d.drawChessboardCorners(bgrMat, new Size((int)squaresX, (int)squaresY), points, found); - } - break; - } - - double[] camMatrixArr = new double[(int)camMatrix.total()]; - camMatrix.get(0, 0, camMatrixArr); - double[] distCoeffsArr = new double[(int)distCoeffs.total()]; - distCoeffs.get(0, 0, distCoeffsArr); - - int ff = Imgproc.FONT_HERSHEY_SIMPLEX; - double fs = 0.4; - Scalar c = new Scalar(255, 255, 255, 255); - int t = 0; - int lt = Imgproc.LINE_AA; - bool blo = false; - int frameCount = (markerType == MarkerType.ChArUcoBoard) ? allCorners.Count : imagePoints.Count; - Imgproc.putText(bgrMat, frameCount + " FRAME CAPTURED", new Point(bgrMat.cols() - 300, 20), ff, fs, c, t, lt, blo); - Imgproc.putText(bgrMat, "IMAGE_WIDTH: " + bgrMat.width(), new Point(bgrMat.cols() - 300, 40), ff, fs, c, t, lt, blo); - Imgproc.putText(bgrMat, "IMAGE_HEIGHT: " + bgrMat.height(), new Point(bgrMat.cols() - 300, 60), ff, fs, c, t, lt, blo); - Imgproc.putText(bgrMat, "CALIBRATION_FLAGS: " + calibrationFlags, new Point(bgrMat.cols() - 300, 80), ff, fs, c, t, lt, blo); - - Imgproc.putText(bgrMat, "CAMERA_MATRIX: ", new Point(bgrMat.cols() - 300, 100), ff, fs, c, t, lt, blo); - for (int i = 0; i < camMatrixArr.Length; i = i + 3) - { - Imgproc.putText(bgrMat, " " + camMatrixArr[i] + ", " + camMatrixArr[i + 1] + ", " + camMatrixArr[i + 2] + ",", new Point(bgrMat.cols() - 300, 120 + 20 * i / 3), ff, fs, c, t, lt, blo); - } - Imgproc.putText(bgrMat, "DISTORTION_COEFFICIENTS: ", new Point(bgrMat.cols() - 300, 180), ff, fs, c, t, lt, blo); - for (int i = 0; i < distCoeffsArr.Length; ++i) - { - Imgproc.putText(bgrMat, " " + distCoeffsArr[i] + ",", new Point(bgrMat.cols() - 300, 200 + 20 * i), ff, fs, c, t, lt, blo); - } - Imgproc.putText(bgrMat, "AVG_REPROJECTION_ERROR: " + repErr, new Point(bgrMat.cols() - 300, 300), ff, fs, c, t, lt, blo); - - if (frameCount == 0) - Imgproc.putText(bgrMat, "To calibration start, please press the calibration button or do air tap gesture!", new Point(5, bgrMat.rows() - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(255, 255, 255, 255), 1, Imgproc.LINE_AA, false); - } - - private double CaptureFrame(Mat frameMat) - { - double repErr = -1; - - switch (markerType) - { - default: - case MarkerType.ChArUcoBoard: - List corners = new List(); - Mat ids = new Mat(); - Aruco.detectMarkers(frameMat, dictionary, corners, ids, detectorParams, rejectedCorners, camMatrix, distCoeffs); - - if (refineMarkerDetection) - { - Aruco.refineDetectedMarkers(frameMat, charucoBoard, corners, ids, rejectedCorners, camMatrix, distCoeffs, 10f, 3f, true, recoveredIdxs, detectorParams); - } - - if (ids.total() > 0) - { - Debug.Log("Frame captured."); - - allCorners.Add(corners); - allIds.Add(ids); - allImgs.Add(frameMat); - } - else - { - - Debug.Log("Invalid frame."); - - frameMat.Dispose(); - if (ids != null) - ids.Dispose(); - foreach (var item in corners) - { - item.Dispose(); - } - corners.Clear(); - - return -1; - } - - // calibrate camera using aruco markers - //double arucoRepErr = CalibrateCameraAruco (allCorners, allIds, charucoBoard, frameMat.size(), camMatrix, distCoeffs, rvecs, tvecs, calibrationFlags); - //Debug.Log ("arucoRepErr: " + arucoRepErr); - - // calibrate camera using charuco - repErr = CalibrateCameraCharuco(allCorners, allIds, charucoBoard, frameMat.size(), camMatrix, distCoeffs, rvecs, tvecs, calibrationFlags, calibrationFlags); - - break; - case MarkerType.ChessBoard: - case MarkerType.CirclesGlid: - case MarkerType.AsymmetricCirclesGlid: - - MatOfPoint2f points = new MatOfPoint2f(); - Size patternSize = new Size((int)squaresX, (int)squaresY); - - bool found = false; - switch (markerType) - { - default: - case MarkerType.ChessBoard: - found = Calib3d.findChessboardCorners(frameMat, patternSize, points, Calib3d.CALIB_CB_ADAPTIVE_THRESH | Calib3d.CALIB_CB_FAST_CHECK | Calib3d.CALIB_CB_NORMALIZE_IMAGE); - break; - case MarkerType.CirclesGlid: - found = Calib3d.findCirclesGrid(frameMat, patternSize, points, Calib3d.CALIB_CB_SYMMETRIC_GRID); - break; - case MarkerType.AsymmetricCirclesGlid: - found = Calib3d.findCirclesGrid(frameMat, patternSize, points, Calib3d.CALIB_CB_ASYMMETRIC_GRID); - break; - } - - if (found) - { - Debug.Log("Frame captured."); - if (markerType == MarkerType.ChessBoard) - Imgproc.cornerSubPix(frameMat, points, new Size(5, 5), new Size(-1, -1), new TermCriteria(TermCriteria.EPS + TermCriteria.COUNT, 30, 0.1)); - - imagePoints.Add(points); - allImgs.Add(frameMat); - } - else - { - Debug.Log("Invalid frame."); - frameMat.Dispose(); - if (points != null) - points.Dispose(); - return -1; - } - - if (imagePoints.Count < 1) - { - Debug.Log("Not enough points for calibration."); - repErr = -1; - } - else - { - - MatOfPoint3f objectPoint = new MatOfPoint3f(new Mat(imagePoints[0].rows(), 1, CvType.CV_32FC3)); - CalcChessboardCorners(patternSize, squareSize, objectPoint, markerType); - - List objectPoints = new List(); - for (int i = 0; i < imagePoints.Count; ++i) - { - objectPoints.Add(objectPoint); - } - - repErr = Calib3d.calibrateCamera(objectPoints, imagePoints, frameMat.size(), camMatrix, distCoeffs, rvecs, tvecs, calibrationFlags); - objectPoint.Dispose(); - } - - break; - } - - Debug.Log("repErr: " + repErr); - Debug.Log("camMatrix: " + camMatrix.dump()); - Debug.Log("distCoeffs: " + distCoeffs.dump()); - - return repErr; - } - - private double CalibrateCameraAruco(List> allCorners, List allIds, CharucoBoard board, Size imageSize, Mat cameraMatrix, Mat distCoeffs, List rvecs = null, List tvecs = null, int calibrationFlags = 0) - { - // prepare data for calibration - int nFrames = allCorners.Count; - int allLen = 0; - int[] markerCounterPerFrameArr = new int[allCorners.Count]; - for (int i = 0; i < nFrames; ++i) - { - markerCounterPerFrameArr[i] = allCorners[i].Count; - allLen += allCorners[i].Count; - } - - int[] allIdsConcatenatedArr = new int[allLen]; - int index = 0; - for (int j = 0; j < allIds.Count; ++j) - { - int[] idsArr = new int[(int)allIds[j].total()]; - allIds[j].get(0, 0, idsArr); - - for (int k = 0; k < idsArr.Length; ++k) - { - allIdsConcatenatedArr[index + k] = (int)idsArr[k]; - } - index += idsArr.Length; - } - - using (Mat allIdsConcatenated = new Mat(1, allLen, CvType.CV_32SC1)) - using (Mat markerCounterPerFrame = new Mat(1, nFrames, CvType.CV_32SC1)) - { - - List allCornersConcatenated = new List(); - foreach (var c in allCorners) - { - foreach (var m in c) - { - allCornersConcatenated.Add(m); - } - } - - allIdsConcatenated.put(0, 0, allIdsConcatenatedArr); - markerCounterPerFrame.put(0, 0, markerCounterPerFrameArr); - - if (rvecs == null) rvecs = new List(); - if (tvecs == null) tvecs = new List(); - - return Aruco.calibrateCameraAruco(allCornersConcatenated, allIdsConcatenated, markerCounterPerFrame, board, imageSize, cameraMatrix, distCoeffs, rvecs, tvecs, calibrationFlags); - } - } - - private double CalibrateCameraCharuco(List> allCorners, List allIds, CharucoBoard board, Size imageSize, Mat cameraMatrix, Mat distCoeffs, List rvecs = null, List tvecs = null, int calibrationFlags = 0, int minMarkers = 2) - { - // prepare data for charuco calibration - int nFrames = allCorners.Count; - List allCharucoCorners = new List(); - List allCharucoIds = new List(); - List filteredImages = new List(); - - for (int i = 0; i < nFrames; ++i) - { - // interpolate using camera parameters - Mat currentCharucoCorners = new Mat(); - Mat currentCharucoIds = new Mat(); - - Aruco.interpolateCornersCharuco(allCorners[i], allIds[i], allImgs[i], board, currentCharucoCorners, currentCharucoIds, cameraMatrix, distCoeffs, minMarkers); - - if (charucoIds.total() > 0) - { - allCharucoCorners.Add(currentCharucoCorners); - allCharucoIds.Add(currentCharucoIds); - filteredImages.Add(allImgs[i]); - } - else - { - currentCharucoCorners.Dispose(); - currentCharucoIds.Dispose(); - } - } - - if (allCharucoCorners.Count < 1) - { - Debug.Log("Not enough corners for calibration."); - return -1; - } - - if (rvecs == null) rvecs = new List(); - if (tvecs == null) tvecs = new List(); - - return Aruco.calibrateCameraCharuco(allCharucoCorners, allCharucoIds, board, imageSize, cameraMatrix, distCoeffs, rvecs, tvecs, calibrationFlags); - } - - private void ResetCalibration() - { - foreach (var corners in allCorners) - { - foreach (var item in corners) - { - item.Dispose(); - } - } - allCorners.Clear(); - - foreach (var item in allIds) - { - item.Dispose(); - } - allIds.Clear(); - - foreach (var item in allImgs) - { - item.Dispose(); - } - allImgs.Clear(); - - repErr = 0; - camMatrix = CreateCameraMatrix(bgrMat.width(), bgrMat.height()); - distCoeffs = new MatOfDouble(0, 0, 0, 0, 0); - - foreach (var item in imagePoints) - { - item.Dispose(); - } - imagePoints.Clear(); - } - - private Mat CreateCameraMatrix(float width, float height) - { - int max_d = (int)Mathf.Max(width, height); - double fx = max_d; - double fy = max_d; - double cx = width / 2.0f; - double cy = height / 2.0f; - - Mat camMatrix = new Mat(3, 3, CvType.CV_64FC1); - camMatrix.put(0, 0, fx); - camMatrix.put(0, 1, 0); - camMatrix.put(0, 2, cx); - camMatrix.put(1, 0, 0); - camMatrix.put(1, 1, fy); - camMatrix.put(1, 2, cy); - camMatrix.put(2, 0, 0); - camMatrix.put(2, 1, 0); - camMatrix.put(2, 2, 1.0f); - - return camMatrix; - } - - private void CalcChessboardCorners(Size patternSize, float squareSize, MatOfPoint3f corners, MarkerType markerType) - { - if ((int)(patternSize.width * patternSize.height) != corners.rows()) - { - Debug.Log("Invalid corners size."); - corners.create((int)(patternSize.width * patternSize.height), 1, CvType.CV_32FC3); - } - - const int cn = 3; - float[] cornersArr = new float[corners.rows() * cn]; - int width = (int)patternSize.width; - int height = (int)patternSize.height; - - switch (markerType) - { - default: - case MarkerType.ChessBoard: - case MarkerType.CirclesGlid: - for (int i = 0; i < height; ++i) - { - for (int j = 0; j < width; ++j) - { - cornersArr[(i * width * cn) + (j * cn)] = j * squareSize; - cornersArr[(i * width * cn) + (j * cn) + 1] = i * squareSize; - cornersArr[(i * width * cn) + (j * cn) + 2] = 0; - } - } - corners.put(0, 0, cornersArr); - - break; - case MarkerType.AsymmetricCirclesGlid: - for (int i = 0; i < height; ++i) - { - for (int j = 0; j < width; ++j) - { - cornersArr[(i * width * cn) + (j * cn)] = (2 * j + i % 2) * squareSize; - cornersArr[(i * width * cn) + (j * cn) + 1] = i * squareSize; - cornersArr[(i * width * cn) + (j * cn) + 2] = 0; - } - } - corners.put(0, 0, cornersArr); - - break; - } - } - - private bool InitializeImagesInputMode() - { - if (isInitialized) - DisposeCalibraton(); - - if (String.IsNullOrEmpty(calibrationImagesDirectory)) - { - Debug.LogWarning("When using the images input mode, please set a calibration images directory path."); - return false; - } - - string dirPath = Path.Combine(Application.streamingAssetsPath, calibrationImagesDirectory); - if (!Directory.Exists(dirPath)) - { - Debug.LogWarning("The directory does not exist."); - return false; - } - string[] imageFiles = GetImageFilesInDirectory(dirPath); - if (imageFiles.Length < 1) - { - Debug.LogWarning("The image file does not exist."); - return false; - } - - Uri rootPath = new Uri(Application.streamingAssetsPath + System.IO.Path.AltDirectorySeparatorChar); - Uri fullPath = new Uri(imageFiles[0]); - string relativePath = rootPath.MakeRelativeUri(fullPath).ToString(); - - using (Mat gray = Imgcodecs.imread(Utils.getFilePath(relativePath), Imgcodecs.IMREAD_GRAYSCALE)) - { - - if (gray.total() == 0) - { - Debug.LogWarning("Invalid image file."); - return false; - } - - using (Mat bgr = new Mat(gray.size(), CvType.CV_8UC3)) - using (Mat bgra = new Mat(gray.size(), CvType.CV_8UC4)) - { - InitializeCalibraton(gray); - - DrawFrame(gray, bgr); - Imgproc.cvtColor(bgr, bgra, Imgproc.COLOR_BGR2RGBA); - Utils.fastMatToTexture2D(bgra, texture); - } - } - return true; - } - - private IEnumerator CalibrateCameraUsingImages() - { - string dirPath = Path.Combine(Application.streamingAssetsPath, calibrationImagesDirectory); - string[] imageFiles = GetImageFilesInDirectory(dirPath); - if (imageFiles.Length < 1) - yield break; - - isCalibrating = true; - markerTypeDropdown.interactable = dictionaryIdDropdown.interactable = squaresXDropdown.interactable = squaresYDropdown.interactable = false; - - Uri rootPath = new Uri(Application.streamingAssetsPath + System.IO.Path.AltDirectorySeparatorChar); - - foreach (var path in imageFiles) - { - - Uri fullPath = new Uri(path); - string relativePath = rootPath.MakeRelativeUri(fullPath).ToString(); - - using (Mat gray = Imgcodecs.imread(Utils.getFilePath(relativePath), Imgcodecs.IMREAD_GRAYSCALE)) - { - - if (gray.width() != bgrMat.width() || gray.height() != bgrMat.height()) - continue; - - Mat frameMat = gray.clone(); - double e = CaptureFrame(frameMat); - if (e > 0) - repErr = e; - - DrawFrame(gray, bgrMat); - Imgproc.cvtColor(bgrMat, rgbaMat, Imgproc.COLOR_BGR2RGBA); - - Utils.matToTexture2D(rgbaMat, texture); - } - yield return new WaitForSeconds(0.5f); - } - - isCalibrating = false; - markerTypeDropdown.interactable = dictionaryIdDropdown.interactable = squaresXDropdown.interactable = squaresYDropdown.interactable = true; - } - - private string[] GetImageFilesInDirectory(string dirPath) - { - if (Directory.Exists(dirPath)) - { - string[] files = Directory.GetFiles(dirPath, "*.jpg"); - files = files.Concat(Directory.GetFiles(dirPath, "*.jpeg")).ToArray(); - files = files.Concat(Directory.GetFiles(dirPath, "*.png")).ToArray(); - files = files.Concat(Directory.GetFiles(dirPath, "*.tiff")).ToArray(); - files = files.Concat(Directory.GetFiles(dirPath, "*.tif")).ToArray(); - return files; - } - return new string[0]; - } - - /// - /// Raises the destroy event. - /// - void OnDestroy() - { - if (isImagesInputMode) - { - DisposeCalibraton(); - } - else - { - webCamTextureToMatHelper.Dispose(); - } - - Screen.orientation = ScreenOrientation.AutoRotation; - } - - /// - /// Raises the back button click event. - /// - public void OnBackButtonClick() - { - SceneManager.LoadScene("HoloLensWithOpenCVForUnityExample"); - } - - /// - /// Raises the play button click event. - /// - public void OnPlayButtonClick() - { - if (isImagesInputMode) return; - - webCamTextureToMatHelper.Play(); - } - - /// - /// Raises the pause button click event. - /// - public void OnPauseButtonClick() - { - if (isImagesInputMode) return; - - webCamTextureToMatHelper.Pause(); - } - - /// - /// Raises the stop button click event. - /// - public void OnStopButtonClick() - { - if (isImagesInputMode) return; - - webCamTextureToMatHelper.Stop(); - } - - /// - /// Raises the change camera button click event. - /// - public void OnChangeCameraButtonClick() - { - if (isImagesInputMode) return; - - webCamTextureToMatHelper.requestedIsFrontFacing = !webCamTextureToMatHelper.IsFrontFacing(); - } - - /// - /// Raises the marker type dropdown value changed event. - /// - public void OnMarkerTypeDropdownValueChanged(int result) - { - if ((int)markerType != result) - { - markerType = (MarkerType)result; - - dictionaryIdDropdown.interactable = (markerType == MarkerType.ChArUcoBoard); - - if (isImagesInputMode) - { - InitializeImagesInputMode(); - } - else - { - if (webCamTextureToMatHelper.IsInitialized()) - webCamTextureToMatHelper.Initialize(); - } - } - } - - /// - /// Raises the dictionary id dropdown value changed event. - /// - public void OnDictionaryIdDropdownValueChanged(int result) - { - if ((int)dictionaryId != result) - { - dictionaryId = (ArUcoDictionary)result; - dictionary = Aruco.getPredefinedDictionary((int)dictionaryId); - - if (isImagesInputMode) - { - InitializeImagesInputMode(); - } - else - { - if (webCamTextureToMatHelper.IsInitialized()) - webCamTextureToMatHelper.Initialize(); - } - } - } - - /// - /// Raises the squares X dropdown value changed event. - /// - public void OnSquaresXDropdownValueChanged(int result) - { - if ((int)squaresX != result + 1) - { - squaresX = (NumberOfSquaresX)(result + 1); - - if (isImagesInputMode) - { - InitializeImagesInputMode(); - } - else - { - if (webCamTextureToMatHelper.IsInitialized()) - webCamTextureToMatHelper.Initialize(); - } - } - } - - /// - /// Raises the squares Y dropdown value changed event. - /// - public void OnSquaresYDropdownValueChanged(int result) - { - if ((int)squaresY != result + 1) - { - squaresY = (NumberOfSquaresY)(result + 1); - - if (isImagesInputMode) - { - InitializeImagesInputMode(); - } - else - { - if (webCamTextureToMatHelper.IsInitialized()) - webCamTextureToMatHelper.Initialize(); - } - } - } - - /// - /// Raises the capture button click event. - /// - public void OnCaptureButtonClick() - { - if (isImagesInputMode) - { - if (!isCalibrating) - InitializeImagesInputMode(); - StartCoroutine("CalibrateCameraUsingImages"); - } - else - { - shouldCaptureFrame = true; - } - } - - /// - /// Raises the reset button click event. - /// - public void OnResetButtonClick() - { - if (isImagesInputMode) - { - if (!isCalibrating) - InitializeImagesInputMode(); - } - else - { - ResetCalibration(); - } - } - - /// - /// Raises the save button click event. - /// - public void OnSaveButtonClick() - { - string saveDirectoryPath = Path.Combine(Application.persistentDataPath, "HoloLensArUcoCameraCalibrationExample"); - - if (!Directory.Exists(saveDirectoryPath)) - { - Directory.CreateDirectory(saveDirectoryPath); - } - - string calibratonDirectoryName = "camera_parameters" + bgrMat.width() + "x" + bgrMat.height(); - string saveCalibratonFileDirectoryPath = Path.Combine(saveDirectoryPath, calibratonDirectoryName); - - // Clean up old files. - if (Directory.Exists(saveCalibratonFileDirectoryPath)) - { - DirectoryInfo directoryInfo = new DirectoryInfo(saveCalibratonFileDirectoryPath); - foreach (FileInfo fileInfo in directoryInfo.GetFiles()) - { - if ((fileInfo.Attributes & FileAttributes.ReadOnly) == FileAttributes.ReadOnly) - { - fileInfo.Attributes = FileAttributes.Normal; - } - } - if ((directoryInfo.Attributes & FileAttributes.ReadOnly) == FileAttributes.ReadOnly) - { - directoryInfo.Attributes = FileAttributes.Directory; - } - directoryInfo.Delete(true); - } - Directory.CreateDirectory(saveCalibratonFileDirectoryPath); - - // save the calibraton file. - string savePath = Path.Combine(saveCalibratonFileDirectoryPath, calibratonDirectoryName + ".xml"); - int frameCount = (markerType == MarkerType.ChArUcoBoard) ? allCorners.Count : imagePoints.Count; - CameraParameters param = new CameraParameters(frameCount, bgrMat.width(), bgrMat.height(), calibrationFlags, camMatrix, distCoeffs, repErr); - XmlSerializer serializer = new XmlSerializer(typeof(CameraParameters)); - using (var stream = new FileStream(savePath, FileMode.Create)) - { - serializer.Serialize(stream, param); - } - - // save the calibration images. -#if UNITY_WEBGL && !UNITY_EDITOR - string format = "jpg"; - MatOfInt compressionParams = new MatOfInt(Imgcodecs.IMWRITE_JPEG_QUALITY, 100); -#else - string format = "png"; - MatOfInt compressionParams = new MatOfInt(Imgcodecs.IMWRITE_PNG_COMPRESSION, 0); -#endif - for (int i = 0; i < allImgs.Count; ++i) - { - Imgcodecs.imwrite(Path.Combine(saveCalibratonFileDirectoryPath, calibratonDirectoryName + "_" + i.ToString("00") + "." + format), allImgs[i], compressionParams); - } - - savePathInputField.text = savePath; - Debug.Log("Saved the CameraParameters to disk in XML file format."); - Debug.Log("savePath: " + savePath); - } - - /// - /// Raises the tapped event. - /// - public void OnTapped(MixedRealityPointerEventData eventData) - { - Debug.Log("OnTapped!"); - - // Determine if a Gaze pointer is over a GUI. - if (eventData.selectedObject != null && (eventData.selectedObject.GetComponent