From ca256644c94956ff7cb97cb5a6801692d4ed8315 Mon Sep 17 00:00:00 2001 From: EnoxSoftware Date: Thu, 2 Jan 2020 19:16:26 +0900 Subject: [PATCH] v1.0.3 MRTKv2 support. Hololens 10.0.17763.914 (RS5) Windows 10 SDK 10.0.18362.0 Visual Studio 2017 or 2019 Unity 2018.4.7f1+ Microsoft Mixed Reality Toolkit v2.2.0 OpenCVForUnity 2.3.7+ HoloLensCameraStream --- .../HoloLensArUcoExample/CameraParameters.cs | 26 +- .../HoloLensArUcoCameraCalibrationExample.cs | 1080 +++++++++-------- ...oloLensArUcoCameraCalibrationExample.unity | 724 ++++++----- .../HoloLensArUcoExample.cs | 581 +++++---- .../HoloLensArUcoExample.meta | 9 - .../HoloLensArUcoExample.unity | 715 +++++------ .../HoloLensArUcoExample/LightingData.asset | Bin 13996 -> 0 bytes .../LightingData.asset.meta | 8 - .../Materials/ArUcoMaterial.mat | 231 ++-- .../HoloLensArUcoExample/TapInput.cs | 33 - .../HoloLensArUcoExample/TapInput.cs.meta | 12 - .../HoloLensComicFilterExample/ComicFilter.cs | 176 +++ .../ComicFilter.cs.meta} | 4 +- .../HoloLensComicFilterExample.cs | 221 +--- .../HoloLensComicFilterExample.unity | 1033 +++++++++++----- .../HoloLensFaceDetectionExample.cs | 392 +++--- .../HoloLensFaceDetectionExample.unity | 675 +++++------ .../HoloLensFaceDetectionOverlayExample.cs | 372 +++--- .../HoloLensFaceDetectionOverlayExample.unity | 645 +++++----- .../HoloLensPhotoCaptureExample.cs | 185 ++- .../HoloLensPhotoCaptureExample.unity | 571 +++++---- .../HoloLensWithOpenCVForUnityExample.cs | 69 +- .../HoloLensWithOpenCVForUnityExample.unity | 576 +++++---- .../holographicImageBlend_material.mat | 2 +- .../Materials/quad_material.mat | 26 +- .../Scripts/ExampleSceneBase.cs | 36 - .../ShowLicense.cs | 17 +- .../ShowLicense.unity | 541 +++++---- .../CameraIntrinsicsChecker.unity | 719 ++++++----- .../CameraIntrinsicsCheckerHelper.cs | 35 +- BuildSettings.jpg | Bin 0 -> 176342 bytes ...df => CanonicalMarker-d10-i1-sp500-bb1.pdf | Bin ChArUcoBoard-mx5-my7-d10-os1000-bb1.pdf | Bin 0 -> 62205 bytes MRTKProjectConfigurator.jpg | Bin 0 -> 83096 bytes PlayerSettings.jpg | Bin 0 -> 411050 bytes ProjectAssets.jpg | Bin 0 -> 169639 bytes README.md | 32 +- buildsetting01.jpg | Bin 100194 -> 0 bytes buildsetting02.jpg | Bin 323041 -> 0 bytes 39 files changed, 5451 insertions(+), 4295 deletions(-) delete mode 100644 Assets/HoloLensWithOpenCVForUnityExample/HoloLensArUcoExample/HoloLensArUcoExample.meta delete mode 100644 Assets/HoloLensWithOpenCVForUnityExample/HoloLensArUcoExample/HoloLensArUcoExample/LightingData.asset delete mode 100644 Assets/HoloLensWithOpenCVForUnityExample/HoloLensArUcoExample/HoloLensArUcoExample/LightingData.asset.meta delete mode 100644 Assets/HoloLensWithOpenCVForUnityExample/HoloLensArUcoExample/TapInput.cs delete mode 100644 Assets/HoloLensWithOpenCVForUnityExample/HoloLensArUcoExample/TapInput.cs.meta create mode 100644 Assets/HoloLensWithOpenCVForUnityExample/HoloLensComicFilterExample/ComicFilter.cs rename Assets/HoloLensWithOpenCVForUnityExample/{Scripts/ExampleSceneBase.cs.meta => HoloLensComicFilterExample/ComicFilter.cs.meta} (71%) delete mode 100644 Assets/HoloLensWithOpenCVForUnityExample/Scripts/ExampleSceneBase.cs create mode 100644 BuildSettings.jpg rename ArUcoMarker_DICT_6X6_250_ID1.pdf => CanonicalMarker-d10-i1-sp500-bb1.pdf (100%) create mode 100644 ChArUcoBoard-mx5-my7-d10-os1000-bb1.pdf create mode 100644 MRTKProjectConfigurator.jpg create mode 100644 PlayerSettings.jpg create mode 100644 ProjectAssets.jpg delete mode 100644 buildsetting01.jpg delete mode 100644 buildsetting02.jpg diff --git a/Assets/HoloLensWithOpenCVForUnityExample/HoloLensArUcoExample/CameraParameters.cs b/Assets/HoloLensWithOpenCVForUnityExample/HoloLensArUcoExample/CameraParameters.cs index 1131547..4efe299 100644 --- a/Assets/HoloLensWithOpenCVForUnityExample/HoloLensArUcoExample/CameraParameters.cs +++ b/Assets/HoloLensWithOpenCVForUnityExample/HoloLensArUcoExample/CameraParameters.cs @@ -6,7 +6,7 @@ namespace HoloLensWithOpenCVForUnityExample [System.Serializable] public struct CameraParameters { - public string calibration_date; + public string calibration_date; public int frames_count; public int image_width; public int image_height; @@ -15,9 +15,9 @@ public struct CameraParameters public double[] distortion_coefficients; public double avg_reprojection_error; - public CameraParameters (int frames_count, int image_width, int image_height, int calibration_flags, double[] camera_matrix, double[] distortion_coefficients, double avg_reprojection_error) + public CameraParameters(int frames_count, int image_width, int image_height, int calibration_flags, double[] camera_matrix, double[] distortion_coefficients, double avg_reprojection_error) { - this.calibration_date = DateTime.Now.ToString (); + this.calibration_date = DateTime.Now.ToString(); this.frames_count = frames_count; this.image_width = image_width; this.image_height = image_height; @@ -27,15 +27,15 @@ public CameraParameters (int frames_count, int image_width, int image_height, in this.avg_reprojection_error = avg_reprojection_error; } - public CameraParameters (int frames_count, int image_width, int image_height, int calibration_flags, Mat camera_matrix, Mat distortion_coefficients, double avg_reprojection_error) + public CameraParameters(int frames_count, int image_width, int image_height, int calibration_flags, Mat camera_matrix, Mat distortion_coefficients, double avg_reprojection_error) { double[] camera_matrixArr = new double[camera_matrix.total()]; - camera_matrix.get (0, 0, camera_matrixArr); + camera_matrix.get(0, 0, camera_matrixArr); double[] distortion_coefficientsArr = new double[distortion_coefficients.total()]; - distortion_coefficients.get (0, 0, distortion_coefficientsArr); + distortion_coefficients.get(0, 0, distortion_coefficientsArr); - this.calibration_date = DateTime.Now.ToString (); + this.calibration_date = DateTime.Now.ToString(); this.frames_count = frames_count; this.image_width = image_width; this.image_height = image_height; @@ -45,17 +45,17 @@ public CameraParameters (int frames_count, int image_width, int image_height, in this.avg_reprojection_error = avg_reprojection_error; } - public Mat GetCameraMatrix () + public Mat GetCameraMatrix() { - Mat m = new Mat (3, 3, CvType.CV_64FC1); - m.put (0, 0, camera_matrix); + Mat m = new Mat(3, 3, CvType.CV_64FC1); + m.put(0, 0, camera_matrix); return m; } - public Mat GetDistortionCoefficients () + public Mat GetDistortionCoefficients() { - Mat m = new Mat (distortion_coefficients.Length, 1, CvType.CV_64FC1); - m.put (0, 0, distortion_coefficients); + Mat m = new Mat(distortion_coefficients.Length, 1, CvType.CV_64FC1); + m.put(0, 0, distortion_coefficients); return m; } } diff --git a/Assets/HoloLensWithOpenCVForUnityExample/HoloLensArUcoExample/HoloLensArUcoCameraCalibrationExample.cs b/Assets/HoloLensWithOpenCVForUnityExample/HoloLensArUcoExample/HoloLensArUcoCameraCalibrationExample.cs index 1d04f21..51e1b04 100644 --- a/Assets/HoloLensWithOpenCVForUnityExample/HoloLensArUcoExample/HoloLensArUcoCameraCalibrationExample.cs +++ b/Assets/HoloLensWithOpenCVForUnityExample/HoloLensArUcoExample/HoloLensArUcoCameraCalibrationExample.cs @@ -1,12 +1,12 @@ using UnityEngine; using UnityEngine.UI; +using UnityEngine.SceneManagement; using System; using System.Collections; using System.Collections.Generic; using System.Xml.Serialization; using System.IO; using System.Linq; -using HoloToolkit.Unity.InputModule; using OpenCVForUnity.CoreModule; using OpenCVForUnity.ArucoModule; using OpenCVForUnity.UnityUtils.Helper; @@ -15,13 +15,7 @@ using OpenCVForUnity.Calib3dModule; using OpenCVForUnity.ImgcodecsModule; using HoloLensWithOpenCVForUnity.UnityUtils.Helper; - -#if UNITY_2017_2_OR_NEWER -using UnityEngine.XR.WSA.Input; -#else -using UnityEngine.VR.WSA.Input; -#endif - +using Microsoft.MixedReality.Toolkit.Input; namespace HoloLensWithOpenCVForUnityExample { @@ -34,7 +28,7 @@ namespace HoloLensWithOpenCVForUnityExample /// https://docs.opencv.org/3.4.0/d7/d21/tutorial_interactive_calibration.html /// [RequireComponent(typeof(HololensCameraStreamToMatHelper))] - public class HoloLensArUcoCameraCalibrationExample : ExampleSceneBase + public class HoloLensArUcoCameraCalibrationExample : MonoBehaviour { /// /// The preview quad. @@ -97,21 +91,21 @@ public class HoloLensArUcoCameraCalibrationExample : ExampleSceneBase /// /// Determines if calibrates camera using the list of calibration images. /// - [TooltipAttribute ("Determines if calibrates camera using the list of calibration images.")] + [TooltipAttribute("Determines if calibrates camera using the list of calibration images.")] public bool isImagesInputMode = false; /// /// The calibration images directory path. /// Set a relative directory path from the starting point of the "StreamingAssets" folder. e.g. "calibration_images/". /// - [TooltipAttribute ("Set a relative directory path from the starting point of the \"StreamingAssets\" folder. e.g. \"calibration_images\"")] + [TooltipAttribute("Set a relative directory path from the starting point of the \"StreamingAssets\" folder. e.g. \"calibration_images\"")] public string calibrationImagesDirectory = "calibration_images"; /// /// The texture. /// Texture2D texture; - + /// /// The webcam texture to mat helper. /// @@ -131,12 +125,12 @@ public class HoloLensArUcoCameraCalibrationExample : ExampleSceneBase /// The rgba mat. /// Mat rgbaMat; - + /// /// The cameraparam matrix. /// Mat camMatrix; - + /// /// The distortion coeffs. /// @@ -146,32 +140,32 @@ public class HoloLensArUcoCameraCalibrationExample : ExampleSceneBase /// The identifiers. /// Mat ids; - + /// /// The corners. /// List corners; - + /// /// The rejected corners. /// List rejectedCorners; - + /// /// The rvecs. /// List rvecs; - + /// /// The tvecs. /// List tvecs; - + /// /// The detector parameters. /// DetectorParameters detectorParams; - + /// /// The dictionary. /// @@ -182,8 +176,6 @@ public class HoloLensArUcoCameraCalibrationExample : ExampleSceneBase /// Mat recoveredIdxs; - GestureRecognizer m_GestureRecognizer; - const int calibrationFlags = 0;// Calib3d.CALIB_FIX_K3 | Calib3d.CALIB_FIX_K4 | Calib3d.CALIB_FIX_K5 double repErr = 0; bool shouldCaptureFrame = false; @@ -209,13 +201,9 @@ public class HoloLensArUcoCameraCalibrationExample : ExampleSceneBase bool isCalibrating = false; // Use this for initialization - new IEnumerator Start () + IEnumerator Start() { - camera = FindObjectOfType(); - cursor = FindObjectOfType(); - input = FindObjectOfType(); - - webCamTextureToMatHelper = gameObject.GetComponent (); + webCamTextureToMatHelper = gameObject.GetComponent(); // fix the screen orientation. Screen.orientation = ScreenOrientation.LandscapeLeft; @@ -223,7 +211,8 @@ public class HoloLensArUcoCameraCalibrationExample : ExampleSceneBase yield return null; if (markerTypeDropdown.value != (int)markerType || dictionaryIdDropdown.value != (int)dictionaryId - || squaresXDropdown.value != (int)squaresX-1 || squaresYDropdown.value != (int)squaresY-1) { + || squaresXDropdown.value != (int)squaresX - 1 || squaresYDropdown.value != (int)squaresY - 1) + { markerTypeDropdown.value = (int)markerType; dictionaryIdDropdown.value = (int)dictionaryId; squaresXDropdown.value = (int)squaresX - 1; @@ -231,530 +220,574 @@ public class HoloLensArUcoCameraCalibrationExample : ExampleSceneBase } dictionaryIdDropdown.interactable = (markerType == MarkerType.ChArUcoBoard); - #if UNITY_WEBGL && !UNITY_EDITOR +#if UNITY_WEBGL && !UNITY_EDITOR isImagesInputMode = false; - #endif - if (isImagesInputMode) { - isImagesInputMode = InitializeImagesInputMode (); +#endif + if (isImagesInputMode) + { + isImagesInputMode = InitializeImagesInputMode(); } - if (!isImagesInputMode) { - webCamTextureToMatHelper.Initialize (); + if (!isImagesInputMode) + { + webCamTextureToMatHelper.Initialize(); } } - + /// /// Raises the webcam texture to mat helper initialized event. /// - public void OnWebCamTextureToMatHelperInitialized () + public void OnWebCamTextureToMatHelperInitialized() { - Debug.Log ("OnWebCamTextureToMatHelperInitialized"); - - Mat webCamTextureMat = webCamTextureToMatHelper.GetMat (); - - InitializeCalibraton (webCamTextureMat); + Debug.Log("OnWebCamTextureToMatHelperInitialized"); + + Mat webCamTextureMat = webCamTextureToMatHelper.GetMat(); + + InitializeCalibraton(webCamTextureMat); // if WebCamera is frontFaceing, flip Mat. - if (webCamTextureToMatHelper.GetWebCamDevice ().isFrontFacing) { + if (webCamTextureToMatHelper.GetWebCamDevice().isFrontFacing) + { webCamTextureToMatHelper.flipHorizontal = true; } - - SetupGestureRecognizer (); } - + /// /// Raises the webcam texture to mat helper disposed event. /// - public void OnWebCamTextureToMatHelperDisposed () + public void OnWebCamTextureToMatHelperDisposed() { - Debug.Log ("OnWebCamTextureToMatHelperDisposed"); - - DisposeCalibraton (); + Debug.Log("OnWebCamTextureToMatHelperDisposed"); - ReleaseGestureRecognizer (); + DisposeCalibraton(); } /// /// Raises the webcam texture to mat helper error occurred event. /// /// Error code. - public void OnWebCamTextureToMatHelperErrorOccurred (WebCamTextureToMatHelper.ErrorCode errorCode) + public void OnWebCamTextureToMatHelperErrorOccurred(WebCamTextureToMatHelper.ErrorCode errorCode) { - Debug.Log ("OnWebCamTextureToMatHelperErrorOccurred " + errorCode); + Debug.Log("OnWebCamTextureToMatHelperErrorOccurred " + errorCode); } - + // Update is called once per frame - void Update () + void Update() { if (isImagesInputMode) return; - if (webCamTextureToMatHelper.IsPlaying () && webCamTextureToMatHelper.DidUpdateThisFrame ()) { + if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) + { - Mat rgbaMat = webCamTextureToMatHelper.GetMat (); + Mat rgbaMat = webCamTextureToMatHelper.GetMat(); - Imgproc.cvtColor (rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY); + Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY); - if (shouldCaptureFrame) { + if (shouldCaptureFrame) + { shouldCaptureFrame = false; - Mat frameMat = grayMat.clone (); - double e = CaptureFrame (frameMat); + Mat frameMat = grayMat.clone(); + double e = CaptureFrame(frameMat); if (e > 0) repErr = e; } - DrawFrame (grayMat, bgrMat); - Imgproc.cvtColor (bgrMat, rgbaMat, Imgproc.COLOR_BGR2RGBA); + DrawFrame(grayMat, bgrMat); + Imgproc.cvtColor(bgrMat, rgbaMat, Imgproc.COLOR_BGR2RGBA); - Utils.fastMatToTexture2D (rgbaMat, texture); + Utils.fastMatToTexture2D(rgbaMat, texture); } } private void InitializeCalibraton(Mat frameMat) { - Debug.Log ("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation); + Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation); - float width = frameMat.width (); - float height = frameMat.height (); + float width = frameMat.width(); + float height = frameMat.height(); - texture = new Texture2D (frameMat.cols (), frameMat.rows (), TextureFormat.RGBA32, false); + texture = new Texture2D(frameMat.cols(), frameMat.rows(), TextureFormat.RGBA32, false); texture.wrapMode = TextureWrapMode.Clamp; previewQuad.GetComponent().material.mainTexture = texture; - previewQuad.transform.localScale = new Vector3 (0.2f * width / height, 0.2f, 1); + previewQuad.transform.localScale = new Vector3(0.2f * width / height, 0.2f, 1); float imageSizeScale = 1.0f; // set cameraparam. - camMatrix = CreateCameraMatrix (width, height); - Debug.Log ("camMatrix " + camMatrix.dump ()); + camMatrix = CreateCameraMatrix(width, height); + Debug.Log("camMatrix " + camMatrix.dump()); - distCoeffs = new MatOfDouble (0, 0, 0, 0, 0); - Debug.Log ("distCoeffs " + distCoeffs.dump ()); + distCoeffs = new MatOfDouble(0, 0, 0, 0, 0); + Debug.Log("distCoeffs " + distCoeffs.dump()); // calibration camera. - Size imageSize = new Size (width * imageSizeScale, height * imageSizeScale); + Size imageSize = new Size(width * imageSizeScale, height * imageSizeScale); double apertureWidth = 0; double apertureHeight = 0; double[] fovx = new double[1]; double[] fovy = new double[1]; double[] focalLength = new double[1]; - Point principalPoint = new Point (0, 0); + Point principalPoint = new Point(0, 0); double[] aspectratio = new double[1]; - Calib3d.calibrationMatrixValues (camMatrix, imageSize, apertureWidth, apertureHeight, fovx, fovy, focalLength, principalPoint, aspectratio); + Calib3d.calibrationMatrixValues(camMatrix, imageSize, apertureWidth, apertureHeight, fovx, fovy, focalLength, principalPoint, aspectratio); - Debug.Log ("imageSize " + imageSize.ToString ()); - Debug.Log ("apertureWidth " + apertureWidth); - Debug.Log ("apertureHeight " + apertureHeight); - Debug.Log ("fovx " + fovx [0]); - Debug.Log ("fovy " + fovy [0]); - Debug.Log ("focalLength " + focalLength [0]); - Debug.Log ("principalPoint " + principalPoint.ToString ()); - Debug.Log ("aspectratio " + aspectratio [0]); + Debug.Log("imageSize " + imageSize.ToString()); + Debug.Log("apertureWidth " + apertureWidth); + Debug.Log("apertureHeight " + apertureHeight); + Debug.Log("fovx " + fovx[0]); + Debug.Log("fovy " + fovy[0]); + Debug.Log("focalLength " + focalLength[0]); + Debug.Log("principalPoint " + principalPoint.ToString()); + Debug.Log("aspectratio " + aspectratio[0]); - grayMat = new Mat (frameMat.rows (), frameMat.cols (), CvType.CV_8UC1); - bgrMat = new Mat (frameMat.rows (), frameMat.cols (), CvType.CV_8UC3); - rgbaMat = new Mat (frameMat.rows (), frameMat.cols (), CvType.CV_8UC4); - ids = new Mat (); - corners = new List (); - rejectedCorners = new List (); - rvecs = new List (); - tvecs = new List (); + grayMat = new Mat(frameMat.rows(), frameMat.cols(), CvType.CV_8UC1); + bgrMat = new Mat(frameMat.rows(), frameMat.cols(), CvType.CV_8UC3); + rgbaMat = new Mat(frameMat.rows(), frameMat.cols(), CvType.CV_8UC4); + ids = new Mat(); + corners = new List(); + rejectedCorners = new List(); + rvecs = new List(); + tvecs = new List(); - detectorParams = DetectorParameters.create (); - detectorParams.set_cornerRefinementMethod (1);// do cornerSubPix() of OpenCV. - dictionary = Aruco.getPredefinedDictionary ((int)dictionaryId); + detectorParams = DetectorParameters.create(); + detectorParams.set_cornerRefinementMethod(1);// do cornerSubPix() of OpenCV. + dictionary = Aruco.getPredefinedDictionary((int)dictionaryId); - recoveredIdxs = new Mat (); + recoveredIdxs = new Mat(); - charucoCorners = new Mat (); - charucoIds = new Mat (); - charucoBoard = CharucoBoard.create ((int)squaresX, (int)squaresY, chArUcoBoradSquareLength, chArUcoBoradMarkerLength, dictionary); + charucoCorners = new Mat(); + charucoIds = new Mat(); + charucoBoard = CharucoBoard.create((int)squaresX, (int)squaresY, chArUcoBoradSquareLength, chArUcoBoradMarkerLength, dictionary); - allCorners = new List> (); - allIds = new List (); - allImgs = new List (); + allCorners = new List>(); + allIds = new List(); + allImgs = new List(); - imagePoints = new List (); + imagePoints = new List(); isInitialized = true; } - private void DisposeCalibraton () + private void DisposeCalibraton() { - ResetCalibration (); + ResetCalibration(); if (grayMat != null) - grayMat.Dispose (); + grayMat.Dispose(); if (bgrMat != null) - bgrMat.Dispose (); + bgrMat.Dispose(); if (rgbaMat != null) - rgbaMat.Dispose (); + rgbaMat.Dispose(); if (ids != null) - ids.Dispose (); - foreach (var item in corners) { - item.Dispose (); + ids.Dispose(); + foreach (var item in corners) + { + item.Dispose(); } - corners.Clear (); - foreach (var item in rejectedCorners) { - item.Dispose (); + corners.Clear(); + foreach (var item in rejectedCorners) + { + item.Dispose(); } - rejectedCorners.Clear (); - foreach (var item in rvecs) { - item.Dispose (); + rejectedCorners.Clear(); + foreach (var item in rvecs) + { + item.Dispose(); } - rvecs.Clear (); - foreach (var item in tvecs) { - item.Dispose (); + rvecs.Clear(); + foreach (var item in tvecs) + { + item.Dispose(); } - tvecs.Clear (); + tvecs.Clear(); if (recoveredIdxs != null) - recoveredIdxs.Dispose (); + recoveredIdxs.Dispose(); if (charucoCorners != null) - charucoCorners.Dispose (); + charucoCorners.Dispose(); if (charucoIds != null) - charucoIds.Dispose (); + charucoIds.Dispose(); if (charucoBoard != null) - charucoBoard.Dispose (); + charucoBoard.Dispose(); isInitialized = false; } - private void DrawFrame (Mat grayMat, Mat bgrMat) + private void DrawFrame(Mat grayMat, Mat bgrMat) { - Imgproc.cvtColor (grayMat, bgrMat, Imgproc.COLOR_GRAY2BGR); - - switch (markerType) { - default: - case MarkerType.ChArUcoBoard: - // detect markers. - Aruco.detectMarkers (grayMat, dictionary, corners, ids, detectorParams, rejectedCorners, camMatrix, distCoeffs); - - // refine marker detection. - if (refineMarkerDetection) { - Aruco.refineDetectedMarkers (grayMat, charucoBoard, corners, ids, rejectedCorners, camMatrix, distCoeffs, 10f, 3f, true, recoveredIdxs, detectorParams); - } + Imgproc.cvtColor(grayMat, bgrMat, Imgproc.COLOR_GRAY2BGR); - // if at least one marker detected - if (ids.total () > 0) { - Aruco.interpolateCornersCharuco(corners, ids, grayMat, charucoBoard, charucoCorners, charucoIds, camMatrix, distCoeffs, charucoMinMarkers); + switch (markerType) + { + default: + case MarkerType.ChArUcoBoard: + // detect markers. + Aruco.detectMarkers(grayMat, dictionary, corners, ids, detectorParams, rejectedCorners, camMatrix, distCoeffs); + + // refine marker detection. + if (refineMarkerDetection) + { + Aruco.refineDetectedMarkers(grayMat, charucoBoard, corners, ids, rejectedCorners, camMatrix, distCoeffs, 10f, 3f, true, recoveredIdxs, detectorParams); + } - // draw markers. - Aruco.drawDetectedMarkers (bgrMat, corners, ids, new Scalar (0, 255, 0, 255)); - // if at least one charuco corner detected - if (charucoIds.total () > 0) { - Aruco.drawDetectedCornersCharuco (bgrMat, charucoCorners, charucoIds, new Scalar(0, 0, 255, 255)); + // if at least one marker detected + if (ids.total() > 0) + { + Aruco.interpolateCornersCharuco(corners, ids, grayMat, charucoBoard, charucoCorners, charucoIds, camMatrix, distCoeffs, charucoMinMarkers); + + // draw markers. + Aruco.drawDetectedMarkers(bgrMat, corners, ids, new Scalar(0, 255, 0, 255)); + // if at least one charuco corner detected + if (charucoIds.total() > 0) + { + Aruco.drawDetectedCornersCharuco(bgrMat, charucoCorners, charucoIds, new Scalar(0, 0, 255, 255)); + } } - } - break; - case MarkerType.ChessBoard: - case MarkerType.CirclesGlid: - case MarkerType.AsymmetricCirclesGlid: - // detect markers. - MatOfPoint2f points = new MatOfPoint2f (); - bool found = false; - - switch (markerType) { - default: - case MarkerType.ChessBoard: - found = Calib3d.findChessboardCorners (grayMat, new Size ((int)squaresX, (int)squaresY), points, Calib3d.CALIB_CB_ADAPTIVE_THRESH | Calib3d.CALIB_CB_FAST_CHECK | Calib3d.CALIB_CB_NORMALIZE_IMAGE); break; + case MarkerType.ChessBoard: case MarkerType.CirclesGlid: - found = Calib3d.findCirclesGrid (grayMat, new Size ((int)squaresX, (int)squaresY), points, Calib3d.CALIB_CB_SYMMETRIC_GRID); - break; case MarkerType.AsymmetricCirclesGlid: - found = Calib3d.findCirclesGrid (grayMat, new Size ((int)squaresX, (int)squaresY), points, Calib3d.CALIB_CB_ASYMMETRIC_GRID); - break; - } + // detect markers. + MatOfPoint2f points = new MatOfPoint2f(); + bool found = false; + + switch (markerType) + { + default: + case MarkerType.ChessBoard: + found = Calib3d.findChessboardCorners(grayMat, new Size((int)squaresX, (int)squaresY), points, Calib3d.CALIB_CB_ADAPTIVE_THRESH | Calib3d.CALIB_CB_FAST_CHECK | Calib3d.CALIB_CB_NORMALIZE_IMAGE); + break; + case MarkerType.CirclesGlid: + found = Calib3d.findCirclesGrid(grayMat, new Size((int)squaresX, (int)squaresY), points, Calib3d.CALIB_CB_SYMMETRIC_GRID); + break; + case MarkerType.AsymmetricCirclesGlid: + found = Calib3d.findCirclesGrid(grayMat, new Size((int)squaresX, (int)squaresY), points, Calib3d.CALIB_CB_ASYMMETRIC_GRID); + break; + } - if (found) { - if (markerType == MarkerType.ChessBoard) - Imgproc.cornerSubPix (grayMat, points, new Size(5, 5), new Size(-1, -1), new TermCriteria(TermCriteria.EPS + TermCriteria.COUNT, 30, 0.1)); + if (found) + { + if (markerType == MarkerType.ChessBoard) + Imgproc.cornerSubPix(grayMat, points, new Size(5, 5), new Size(-1, -1), new TermCriteria(TermCriteria.EPS + TermCriteria.COUNT, 30, 0.1)); - // draw markers. - Calib3d.drawChessboardCorners (bgrMat, new Size ((int)squaresX, (int)squaresY), points, found); - } - break; + // draw markers. + Calib3d.drawChessboardCorners(bgrMat, new Size((int)squaresX, (int)squaresY), points, found); + } + break; } - double[] camMatrixArr = new double[(int)camMatrix.total ()]; - camMatrix.get (0, 0, camMatrixArr); - double[] distCoeffsArr = new double[(int)distCoeffs.total ()]; - distCoeffs.get (0, 0, distCoeffsArr); + double[] camMatrixArr = new double[(int)camMatrix.total()]; + camMatrix.get(0, 0, camMatrixArr); + double[] distCoeffsArr = new double[(int)distCoeffs.total()]; + distCoeffs.get(0, 0, distCoeffsArr); int ff = Imgproc.FONT_HERSHEY_SIMPLEX; double fs = 0.4; - Scalar c = new Scalar (255, 255, 255, 255); + Scalar c = new Scalar(255, 255, 255, 255); int t = 0; int lt = Imgproc.LINE_AA; bool blo = false; int frameCount = (markerType == MarkerType.ChArUcoBoard) ? allCorners.Count : imagePoints.Count; - Imgproc.putText (bgrMat, frameCount + " FRAME CAPTURED", new Point (bgrMat.cols() - 300, 20), ff, fs, c, t, lt, blo); - Imgproc.putText (bgrMat, "IMAGE_WIDTH: " + bgrMat.width(), new Point (bgrMat.cols() - 300, 40), ff, fs, c, t, lt, blo); - Imgproc.putText (bgrMat, "IMAGE_HEIGHT: " + bgrMat.height(), new Point (bgrMat.cols() - 300, 60), ff, fs, c, t, lt, blo); - Imgproc.putText (bgrMat, "CALIBRATION_FLAGS: " + calibrationFlags, new Point (bgrMat.cols() - 300, 80), ff, fs, c, t, lt, blo); - - Imgproc.putText (bgrMat, "CAMERA_MATRIX: ", new Point (bgrMat.cols() - 300, 100), ff, fs, c, t, lt, blo); - for (int i = 0; i < camMatrixArr.Length; i=i+3) { - Imgproc.putText (bgrMat, " " + camMatrixArr [i] + ", " + camMatrixArr [i+1] + ", " + camMatrixArr [i+2] + ",", new Point (bgrMat.cols () - 300, 120+20*i/3), ff, fs, c, t, lt, blo); + Imgproc.putText(bgrMat, frameCount + " FRAME CAPTURED", new Point(bgrMat.cols() - 300, 20), ff, fs, c, t, lt, blo); + Imgproc.putText(bgrMat, "IMAGE_WIDTH: " + bgrMat.width(), new Point(bgrMat.cols() - 300, 40), ff, fs, c, t, lt, blo); + Imgproc.putText(bgrMat, "IMAGE_HEIGHT: " + bgrMat.height(), new Point(bgrMat.cols() - 300, 60), ff, fs, c, t, lt, blo); + Imgproc.putText(bgrMat, "CALIBRATION_FLAGS: " + calibrationFlags, new Point(bgrMat.cols() - 300, 80), ff, fs, c, t, lt, blo); + + Imgproc.putText(bgrMat, "CAMERA_MATRIX: ", new Point(bgrMat.cols() - 300, 100), ff, fs, c, t, lt, blo); + for (int i = 0; i < camMatrixArr.Length; i = i + 3) + { + Imgproc.putText(bgrMat, " " + camMatrixArr[i] + ", " + camMatrixArr[i + 1] + ", " + camMatrixArr[i + 2] + ",", new Point(bgrMat.cols() - 300, 120 + 20 * i / 3), ff, fs, c, t, lt, blo); } - Imgproc.putText (bgrMat, "DISTORTION_COEFFICIENTS: ", new Point (bgrMat.cols() - 300, 180), ff, fs, c, t, lt, blo); - for (int i = 0; i < distCoeffsArr.Length; ++i) { - Imgproc.putText (bgrMat, " " + distCoeffsArr [i] + ",", new Point (bgrMat.cols () - 300, 200+20*i), ff, fs, c, t, lt, blo); + Imgproc.putText(bgrMat, "DISTORTION_COEFFICIENTS: ", new Point(bgrMat.cols() - 300, 180), ff, fs, c, t, lt, blo); + for (int i = 0; i < distCoeffsArr.Length; ++i) + { + Imgproc.putText(bgrMat, " " + distCoeffsArr[i] + ",", new Point(bgrMat.cols() - 300, 200 + 20 * i), ff, fs, c, t, lt, blo); } - Imgproc.putText (bgrMat, "AVG_REPROJECTION_ERROR: " + repErr, new Point (bgrMat.cols() - 300, 300), ff, fs, c, t, lt, blo); + Imgproc.putText(bgrMat, "AVG_REPROJECTION_ERROR: " + repErr, new Point(bgrMat.cols() - 300, 300), ff, fs, c, t, lt, blo); if (frameCount == 0) - Imgproc.putText (bgrMat, "To calibration start, please press the calibration button or do air tap gesture!", new Point (5, bgrMat.rows () - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 1, Imgproc.LINE_AA, false); + Imgproc.putText(bgrMat, "To calibration start, please press the calibration button or do air tap gesture!", new Point(5, bgrMat.rows() - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(255, 255, 255, 255), 1, Imgproc.LINE_AA, false); } - private double CaptureFrame (Mat frameMat) + private double CaptureFrame(Mat frameMat) { double repErr = -1; - switch (markerType) { - default: - case MarkerType.ChArUcoBoard: - List corners = new List (); - Mat ids = new Mat (); - Aruco.detectMarkers (frameMat, dictionary, corners, ids, detectorParams, rejectedCorners, camMatrix, distCoeffs); + switch (markerType) + { + default: + case MarkerType.ChArUcoBoard: + List corners = new List(); + Mat ids = new Mat(); + Aruco.detectMarkers(frameMat, dictionary, corners, ids, detectorParams, rejectedCorners, camMatrix, distCoeffs); + + if (refineMarkerDetection) + { + Aruco.refineDetectedMarkers(frameMat, charucoBoard, corners, ids, rejectedCorners, camMatrix, distCoeffs, 10f, 3f, true, recoveredIdxs, detectorParams); + } - if (refineMarkerDetection) { - Aruco.refineDetectedMarkers (frameMat, charucoBoard, corners, ids, rejectedCorners, camMatrix, distCoeffs, 10f, 3f, true, recoveredIdxs, detectorParams); - } + if (ids.total() > 0) + { + Debug.Log("Frame captured."); - if (ids.total () > 0) { - Debug.Log ("Frame captured."); + allCorners.Add(corners); + allIds.Add(ids); + allImgs.Add(frameMat); + } + else + { - allCorners.Add (corners); - allIds.Add (ids); - allImgs.Add (frameMat); - } else { + Debug.Log("Invalid frame."); - Debug.Log ("Invalid frame."); + frameMat.Dispose(); + if (ids != null) + ids.Dispose(); + foreach (var item in corners) + { + item.Dispose(); + } + corners.Clear(); - frameMat.Dispose (); - if (ids != null) - ids.Dispose (); - foreach (var item in corners) { - item.Dispose (); + return -1; } - corners.Clear (); - return -1; - } + // calibrate camera using aruco markers + //double arucoRepErr = CalibrateCameraAruco (allCorners, allIds, charucoBoard, frameMat.size(), camMatrix, distCoeffs, rvecs, tvecs, calibrationFlags); + //Debug.Log ("arucoRepErr: " + arucoRepErr); - // calibrate camera using aruco markers - //double arucoRepErr = CalibrateCameraAruco (allCorners, allIds, charucoBoard, frameMat.size(), camMatrix, distCoeffs, rvecs, tvecs, calibrationFlags); - //Debug.Log ("arucoRepErr: " + arucoRepErr); + // calibrate camera using charuco + repErr = CalibrateCameraCharuco(allCorners, allIds, charucoBoard, frameMat.size(), camMatrix, distCoeffs, rvecs, tvecs, calibrationFlags, calibrationFlags); - // calibrate camera using charuco - repErr = CalibrateCameraCharuco (allCorners, allIds, charucoBoard, frameMat.size(), camMatrix, distCoeffs, rvecs, tvecs, calibrationFlags, calibrationFlags); - - break; - case MarkerType.ChessBoard: - case MarkerType.CirclesGlid: - case MarkerType.AsymmetricCirclesGlid: - - MatOfPoint2f points = new MatOfPoint2f (); - Size patternSize = new Size ((int)squaresX, (int)squaresY); - - bool found = false; - switch (markerType) { - default: - case MarkerType.ChessBoard: - found = Calib3d.findChessboardCorners (frameMat, patternSize, points, Calib3d.CALIB_CB_ADAPTIVE_THRESH | Calib3d.CALIB_CB_FAST_CHECK | Calib3d.CALIB_CB_NORMALIZE_IMAGE); break; + case MarkerType.ChessBoard: case MarkerType.CirclesGlid: - found = Calib3d.findCirclesGrid (frameMat, patternSize, points, Calib3d.CALIB_CB_SYMMETRIC_GRID); - break; case MarkerType.AsymmetricCirclesGlid: - found = Calib3d.findCirclesGrid (frameMat, patternSize, points, Calib3d.CALIB_CB_ASYMMETRIC_GRID); - break; - } - if (found) { - Debug.Log ("Frame captured."); - if (markerType == MarkerType.ChessBoard) - Imgproc.cornerSubPix (frameMat, points, new Size(5, 5), new Size(-1, -1), new TermCriteria(TermCriteria.EPS + TermCriteria.COUNT, 30, 0.1)); - - imagePoints.Add (points); - allImgs.Add (frameMat); - } else { - Debug.Log ("Invalid frame."); - frameMat.Dispose (); - if (points != null) - points.Dispose (); - return -1; - } - - if (imagePoints.Count < 1) { - Debug.Log ("Not enough points for calibration."); - repErr = -1; - } else { - - MatOfPoint3f objectPoint = new MatOfPoint3f (new Mat (imagePoints [0].rows (), 1, CvType.CV_32FC3)); - CalcChessboardCorners (patternSize, squareSize, objectPoint, markerType); - - List objectPoints = new List (); - for (int i = 0; i < imagePoints.Count; ++i) { - objectPoints.Add (objectPoint); + MatOfPoint2f points = new MatOfPoint2f(); + Size patternSize = new Size((int)squaresX, (int)squaresY); + + bool found = false; + switch (markerType) + { + default: + case MarkerType.ChessBoard: + found = Calib3d.findChessboardCorners(frameMat, patternSize, points, Calib3d.CALIB_CB_ADAPTIVE_THRESH | Calib3d.CALIB_CB_FAST_CHECK | Calib3d.CALIB_CB_NORMALIZE_IMAGE); + break; + case MarkerType.CirclesGlid: + found = Calib3d.findCirclesGrid(frameMat, patternSize, points, Calib3d.CALIB_CB_SYMMETRIC_GRID); + break; + case MarkerType.AsymmetricCirclesGlid: + found = Calib3d.findCirclesGrid(frameMat, patternSize, points, Calib3d.CALIB_CB_ASYMMETRIC_GRID); + break; } - repErr = Calib3d.calibrateCamera (objectPoints, imagePoints, frameMat.size (), camMatrix, distCoeffs, rvecs, tvecs, calibrationFlags); - objectPoint.Dispose (); - } + if (found) + { + Debug.Log("Frame captured."); + if (markerType == MarkerType.ChessBoard) + Imgproc.cornerSubPix(frameMat, points, new Size(5, 5), new Size(-1, -1), new TermCriteria(TermCriteria.EPS + TermCriteria.COUNT, 30, 0.1)); + + imagePoints.Add(points); + allImgs.Add(frameMat); + } + else + { + Debug.Log("Invalid frame."); + frameMat.Dispose(); + if (points != null) + points.Dispose(); + return -1; + } + + if (imagePoints.Count < 1) + { + Debug.Log("Not enough points for calibration."); + repErr = -1; + } + else + { + + MatOfPoint3f objectPoint = new MatOfPoint3f(new Mat(imagePoints[0].rows(), 1, CvType.CV_32FC3)); + CalcChessboardCorners(patternSize, squareSize, objectPoint, markerType); - break; + List objectPoints = new List(); + for (int i = 0; i < imagePoints.Count; ++i) + { + objectPoints.Add(objectPoint); + } + + repErr = Calib3d.calibrateCamera(objectPoints, imagePoints, frameMat.size(), camMatrix, distCoeffs, rvecs, tvecs, calibrationFlags); + objectPoint.Dispose(); + } + + break; } - Debug.Log ("repErr: " + repErr); - Debug.Log ("camMatrix: " + camMatrix.dump ()); - Debug.Log ("distCoeffs: " + distCoeffs.dump ()); - + Debug.Log("repErr: " + repErr); + Debug.Log("camMatrix: " + camMatrix.dump()); + Debug.Log("distCoeffs: " + distCoeffs.dump()); + return repErr; } - private double CalibrateCameraAruco (List> allCorners, List allIds, CharucoBoard board, Size imageSize, Mat cameraMatrix, Mat distCoeffs, List rvecs = null, List tvecs = null, int calibrationFlags = 0) + private double CalibrateCameraAruco(List> allCorners, List allIds, CharucoBoard board, Size imageSize, Mat cameraMatrix, Mat distCoeffs, List rvecs = null, List tvecs = null, int calibrationFlags = 0) { // prepare data for calibration int nFrames = allCorners.Count; int allLen = 0; int[] markerCounterPerFrameArr = new int[allCorners.Count]; - for(int i=0; i allCornersConcatenated = new List (); - foreach (var c in allCorners) { - foreach (var m in c) { - allCornersConcatenated.Add (m); + List allCornersConcatenated = new List(); + foreach (var c in allCorners) + { + foreach (var m in c) + { + allCornersConcatenated.Add(m); } } - allIdsConcatenated.put (0, 0, allIdsConcatenatedArr); - markerCounterPerFrame.put (0, 0, markerCounterPerFrameArr); + allIdsConcatenated.put(0, 0, allIdsConcatenatedArr); + markerCounterPerFrame.put(0, 0, markerCounterPerFrameArr); - if (rvecs == null) rvecs = new List (); - if (tvecs == null) tvecs = new List (); + if (rvecs == null) rvecs = new List(); + if (tvecs == null) tvecs = new List(); - return Aruco.calibrateCameraAruco (allCornersConcatenated, allIdsConcatenated, markerCounterPerFrame, board, imageSize, cameraMatrix, distCoeffs, rvecs, tvecs, calibrationFlags); + return Aruco.calibrateCameraAruco(allCornersConcatenated, allIdsConcatenated, markerCounterPerFrame, board, imageSize, cameraMatrix, distCoeffs, rvecs, tvecs, calibrationFlags); } } - private double CalibrateCameraCharuco (List> allCorners, List allIds, CharucoBoard board, Size imageSize, Mat cameraMatrix, Mat distCoeffs, List rvecs = null, List tvecs = null, int calibrationFlags = 0, int minMarkers = 2) + private double CalibrateCameraCharuco(List> allCorners, List allIds, CharucoBoard board, Size imageSize, Mat cameraMatrix, Mat distCoeffs, List rvecs = null, List tvecs = null, int calibrationFlags = 0, int minMarkers = 2) { // prepare data for charuco calibration int nFrames = allCorners.Count; - List allCharucoCorners = new List (); - List allCharucoIds = new List (); - List filteredImages = new List (); + List allCharucoCorners = new List(); + List allCharucoIds = new List(); + List filteredImages = new List(); - for (int i = 0; i < nFrames; ++i) { + for (int i = 0; i < nFrames; ++i) + { // interpolate using camera parameters Mat currentCharucoCorners = new Mat(); Mat currentCharucoIds = new Mat(); - Aruco.interpolateCornersCharuco (allCorners[i], allIds[i], allImgs[i], board, currentCharucoCorners, currentCharucoIds, cameraMatrix, distCoeffs, minMarkers); + Aruco.interpolateCornersCharuco(allCorners[i], allIds[i], allImgs[i], board, currentCharucoCorners, currentCharucoIds, cameraMatrix, distCoeffs, minMarkers); - if (charucoIds.total () > 0) { - allCharucoCorners.Add (currentCharucoCorners); - allCharucoIds.Add (currentCharucoIds); - filteredImages.Add (allImgs[i]); - } else { - currentCharucoCorners.Dispose (); - currentCharucoIds.Dispose (); + if (charucoIds.total() > 0) + { + allCharucoCorners.Add(currentCharucoCorners); + allCharucoIds.Add(currentCharucoIds); + filteredImages.Add(allImgs[i]); + } + else + { + currentCharucoCorners.Dispose(); + currentCharucoIds.Dispose(); } } - if (allCharucoCorners.Count < 1){ - Debug.Log ("Not enough corners for calibration."); + if (allCharucoCorners.Count < 1) + { + Debug.Log("Not enough corners for calibration."); return -1; } - if (rvecs == null) rvecs = new List (); - if (tvecs == null) tvecs = new List (); + if (rvecs == null) rvecs = new List(); + if (tvecs == null) tvecs = new List(); - return Aruco.calibrateCameraCharuco (allCharucoCorners, allCharucoIds, board, imageSize, cameraMatrix, distCoeffs, rvecs, tvecs, calibrationFlags); + return Aruco.calibrateCameraCharuco(allCharucoCorners, allCharucoIds, board, imageSize, cameraMatrix, distCoeffs, rvecs, tvecs, calibrationFlags); } - private void ResetCalibration () + private void ResetCalibration() { - foreach (var corners in allCorners) { - foreach (var item in corners) { - item.Dispose (); + foreach (var corners in allCorners) + { + foreach (var item in corners) + { + item.Dispose(); } } - allCorners.Clear (); + allCorners.Clear(); - foreach (var item in allIds) { - item.Dispose (); + foreach (var item in allIds) + { + item.Dispose(); } - allIds.Clear (); + allIds.Clear(); - foreach (var item in allImgs) { - item.Dispose (); + foreach (var item in allImgs) + { + item.Dispose(); } - allImgs.Clear (); + allImgs.Clear(); repErr = 0; - camMatrix = CreateCameraMatrix (bgrMat.width(), bgrMat.height()); - distCoeffs = new MatOfDouble (0, 0, 0, 0, 0); + camMatrix = CreateCameraMatrix(bgrMat.width(), bgrMat.height()); + distCoeffs = new MatOfDouble(0, 0, 0, 0, 0); - foreach (var item in imagePoints) { - item.Dispose (); + foreach (var item in imagePoints) + { + item.Dispose(); } - imagePoints.Clear (); + imagePoints.Clear(); } - private Mat CreateCameraMatrix (float width, float height) + private Mat CreateCameraMatrix(float width, float height) { - int max_d = (int)Mathf.Max (width, height); + int max_d = (int)Mathf.Max(width, height); double fx = max_d; double fy = max_d; double cx = width / 2.0f; double cy = height / 2.0f; - Mat camMatrix = new Mat (3, 3, CvType.CV_64FC1); - camMatrix.put (0, 0, fx); - camMatrix.put (0, 1, 0); - camMatrix.put (0, 2, cx); - camMatrix.put (1, 0, 0); - camMatrix.put (1, 1, fy); - camMatrix.put (1, 2, cy); - camMatrix.put (2, 0, 0); - camMatrix.put (2, 1, 0); - camMatrix.put (2, 2, 1.0f); + Mat camMatrix = new Mat(3, 3, CvType.CV_64FC1); + camMatrix.put(0, 0, fx); + camMatrix.put(0, 1, 0); + camMatrix.put(0, 2, cx); + camMatrix.put(1, 0, 0); + camMatrix.put(1, 1, fy); + camMatrix.put(1, 2, cy); + camMatrix.put(2, 0, 0); + camMatrix.put(2, 1, 0); + camMatrix.put(2, 2, 1.0f); return camMatrix; } - private void CalcChessboardCorners (Size patternSize, float squareSize, MatOfPoint3f corners, MarkerType markerType) + private void CalcChessboardCorners(Size patternSize, float squareSize, MatOfPoint3f corners, MarkerType markerType) { - if ((int)(patternSize.width * patternSize.height) != corners.rows()) { - Debug.Log ("Invalid corners size."); - corners.create ((int)(patternSize.width * patternSize.height), 1, CvType.CV_32FC3); + if ((int)(patternSize.width * patternSize.height) != corners.rows()) + { + Debug.Log("Invalid corners size."); + corners.create((int)(patternSize.width * patternSize.height), 1, CvType.CV_32FC3); } const int cn = 3; @@ -762,82 +795,93 @@ private void CalcChessboardCorners (Size patternSize, float squareSize, MatOfPoi int width = (int)patternSize.width; int height = (int)patternSize.height; - switch (markerType) { - default: - case MarkerType.ChessBoard: - case MarkerType.CirclesGlid: - for (int i = 0; i < height; ++i) { - for (int j = 0; j < width; ++j) { - cornersArr [(i * width * cn) + (j * cn)] = j * squareSize; - cornersArr [(i * width * cn) + (j * cn) + 1] = i * squareSize; - cornersArr [(i * width * cn) + (j * cn) + 2] = 0; + switch (markerType) + { + default: + case MarkerType.ChessBoard: + case MarkerType.CirclesGlid: + for (int i = 0; i < height; ++i) + { + for (int j = 0; j < width; ++j) + { + cornersArr[(i * width * cn) + (j * cn)] = j * squareSize; + cornersArr[(i * width * cn) + (j * cn) + 1] = i * squareSize; + cornersArr[(i * width * cn) + (j * cn) + 2] = 0; + } } - } - corners.put (0, 0, cornersArr); - - break; - case MarkerType.AsymmetricCirclesGlid: - for (int i = 0; i < height; ++i) { - for (int j = 0; j < width; ++j) { - cornersArr [(i * width * cn) + (j * cn)] = (2*j + i%2) * squareSize; - cornersArr [(i * width * cn) + (j * cn) + 1] = i * squareSize; - cornersArr [(i * width * cn) + (j * cn) + 2] = 0; + corners.put(0, 0, cornersArr); + + break; + case MarkerType.AsymmetricCirclesGlid: + for (int i = 0; i < height; ++i) + { + for (int j = 0; j < width; ++j) + { + cornersArr[(i * width * cn) + (j * cn)] = (2 * j + i % 2) * squareSize; + cornersArr[(i * width * cn) + (j * cn) + 1] = i * squareSize; + cornersArr[(i * width * cn) + (j * cn) + 2] = 0; + } } - } - corners.put (0, 0, cornersArr); + corners.put(0, 0, cornersArr); - break; + break; } } - private bool InitializeImagesInputMode () + private bool InitializeImagesInputMode() { if (isInitialized) - DisposeCalibraton (); + DisposeCalibraton(); - if (String.IsNullOrEmpty (calibrationImagesDirectory)) { - Debug.LogWarning ("When using the images input mode, please set a calibration images directory path."); + if (String.IsNullOrEmpty(calibrationImagesDirectory)) + { + Debug.LogWarning("When using the images input mode, please set a calibration images directory path."); return false; } - string dirPath = Path.Combine (Application.streamingAssetsPath, calibrationImagesDirectory); - if (!Directory.Exists (dirPath)) { - Debug.LogWarning ("The directory does not exist."); + string dirPath = Path.Combine(Application.streamingAssetsPath, calibrationImagesDirectory); + if (!Directory.Exists(dirPath)) + { + Debug.LogWarning("The directory does not exist."); return false; } - string[] imageFiles = GetImageFilesInDirectory (dirPath); - if (imageFiles.Length < 1) { - Debug.LogWarning ("The image file does not exist."); + string[] imageFiles = GetImageFilesInDirectory(dirPath); + if (imageFiles.Length < 1) + { + Debug.LogWarning("The image file does not exist."); return false; - } - + } + Uri rootPath = new Uri(Application.streamingAssetsPath + System.IO.Path.AltDirectorySeparatorChar); Uri fullPath = new Uri(imageFiles[0]); string relativePath = rootPath.MakeRelativeUri(fullPath).ToString(); - using (Mat gray = Imgcodecs.imread (Utils.getFilePath(relativePath), Imgcodecs.IMREAD_GRAYSCALE)) { + using (Mat gray = Imgcodecs.imread(Utils.getFilePath(relativePath), Imgcodecs.IMREAD_GRAYSCALE)) + { - if (gray.total () == 0) { - Debug.LogWarning ("Invalid image file."); + if (gray.total() == 0) + { + Debug.LogWarning("Invalid image file."); return false; } - using (Mat bgr = new Mat (gray.size (), CvType.CV_8UC3)) - using (Mat bgra = new Mat (gray.size (), CvType.CV_8UC4)) { - InitializeCalibraton (gray); + using (Mat bgr = new Mat(gray.size(), CvType.CV_8UC3)) + using (Mat bgra = new Mat(gray.size(), CvType.CV_8UC4)) + { + InitializeCalibraton(gray); - DrawFrame (gray, bgr); - Imgproc.cvtColor (bgr, bgra, Imgproc.COLOR_BGR2RGBA); - Utils.fastMatToTexture2D (bgra, texture); + DrawFrame(gray, bgr); + Imgproc.cvtColor(bgr, bgra, Imgproc.COLOR_BGR2RGBA); + Utils.fastMatToTexture2D(bgra, texture); } } return true; } - private IEnumerator CalibrateCameraUsingImages () + private IEnumerator CalibrateCameraUsingImages() { - string dirPath = Path.Combine (Application.streamingAssetsPath, calibrationImagesDirectory); - string[] imageFiles = GetImageFilesInDirectory (dirPath); + string dirPath = Path.Combine(Application.streamingAssetsPath, calibrationImagesDirectory); + string[] imageFiles = GetImageFilesInDirectory(dirPath); if (imageFiles.Length < 1) yield break; @@ -846,25 +890,27 @@ private IEnumerator CalibrateCameraUsingImages () Uri rootPath = new Uri(Application.streamingAssetsPath + System.IO.Path.AltDirectorySeparatorChar); - foreach (var path in imageFiles){ + foreach (var path in imageFiles) + { Uri fullPath = new Uri(path); string relativePath = rootPath.MakeRelativeUri(fullPath).ToString(); - using (Mat gray = Imgcodecs.imread (Utils.getFilePath(relativePath), Imgcodecs.IMREAD_GRAYSCALE)) { + using (Mat gray = Imgcodecs.imread(Utils.getFilePath(relativePath), Imgcodecs.IMREAD_GRAYSCALE)) + { - if (gray.width() != bgrMat.width () || gray.height() != bgrMat.height ()) + if (gray.width() != bgrMat.width() || gray.height() != bgrMat.height()) continue; - Mat frameMat = gray.clone (); - double e = CaptureFrame (frameMat); + Mat frameMat = gray.clone(); + double e = CaptureFrame(frameMat); if (e > 0) repErr = e; - DrawFrame (gray, bgrMat); - Imgproc.cvtColor (bgrMat, rgbaMat, Imgproc.COLOR_BGR2RGBA); + DrawFrame(gray, bgrMat); + Imgproc.cvtColor(bgrMat, rgbaMat, Imgproc.COLOR_BGR2RGBA); - Utils.matToTexture2D (rgbaMat, texture); + Utils.matToTexture2D(rgbaMat, texture); } yield return new WaitForSeconds(0.5f); } @@ -873,14 +919,15 @@ private IEnumerator CalibrateCameraUsingImages () markerTypeDropdown.interactable = dictionaryIdDropdown.interactable = squaresXDropdown.interactable = squaresYDropdown.interactable = true; } - private string[] GetImageFilesInDirectory (string dirPath) + private string[] GetImageFilesInDirectory(string dirPath) { - if (Directory.Exists (dirPath)) { - string[] files = Directory.GetFiles (dirPath, "*.jpg"); - files = files.Concat ( Directory.GetFiles (dirPath, "*.jpeg")).ToArray(); - files = files.Concat ( Directory.GetFiles (dirPath, "*.png")).ToArray(); - files = files.Concat ( Directory.GetFiles (dirPath, "*.tiff")).ToArray(); - files = files.Concat ( Directory.GetFiles (dirPath, "*.tif")).ToArray(); + if (Directory.Exists(dirPath)) + { + string[] files = Directory.GetFiles(dirPath, "*.jpg"); + files = files.Concat(Directory.GetFiles(dirPath, "*.jpeg")).ToArray(); + files = files.Concat(Directory.GetFiles(dirPath, "*.png")).ToArray(); + files = files.Concat(Directory.GetFiles(dirPath, "*.tiff")).ToArray(); + files = files.Concat(Directory.GetFiles(dirPath, "*.tif")).ToArray(); return files; } return new string[0]; @@ -889,59 +936,62 @@ private string[] GetImageFilesInDirectory (string dirPath) /// /// Raises the destroy event. /// - void OnDestroy () + void OnDestroy() { - if (isImagesInputMode) { - DisposeCalibraton (); - } else { - webCamTextureToMatHelper.Dispose (); + if (isImagesInputMode) + { + DisposeCalibraton(); } - + else + { + webCamTextureToMatHelper.Dispose(); + } + Screen.orientation = ScreenOrientation.AutoRotation; } - + /// /// Raises the back button click event. /// - public void OnBackButtonClick () + public void OnBackButtonClick() { - LoadScene ("HoloLensWithOpenCVForUnityExample"); + SceneManager.LoadScene("HoloLensWithOpenCVForUnityExample"); } - + /// /// Raises the play button click event. /// - public void OnPlayButtonClick () + public void OnPlayButtonClick() { if (isImagesInputMode) return; - webCamTextureToMatHelper.Play (); + webCamTextureToMatHelper.Play(); } - + /// /// Raises the pause button click event. /// - public void OnPauseButtonClick () + public void OnPauseButtonClick() { if (isImagesInputMode) return; - webCamTextureToMatHelper.Pause (); + webCamTextureToMatHelper.Pause(); } - + /// /// Raises the stop button click event. /// - public void OnStopButtonClick () + public void OnStopButtonClick() { if (isImagesInputMode) return; - webCamTextureToMatHelper.Stop (); + webCamTextureToMatHelper.Stop(); } - + /// /// Raises the change camera button click event. /// - public void OnChangeCameraButtonClick () + public void OnChangeCameraButtonClick() { if (isImagesInputMode) return; @@ -953,16 +1003,20 @@ public void OnChangeCameraButtonClick () /// public void OnMarkerTypeDropdownValueChanged(int result) { - if ((int)markerType != result) { + if ((int)markerType != result) + { markerType = (MarkerType)result; dictionaryIdDropdown.interactable = (markerType == MarkerType.ChArUcoBoard); - if (isImagesInputMode) { - InitializeImagesInputMode (); - } else { - if (webCamTextureToMatHelper.IsInitialized ()) - webCamTextureToMatHelper.Initialize (); + if (isImagesInputMode) + { + InitializeImagesInputMode(); + } + else + { + if (webCamTextureToMatHelper.IsInitialized()) + webCamTextureToMatHelper.Initialize(); } } } @@ -972,15 +1026,19 @@ public void OnMarkerTypeDropdownValueChanged(int result) /// public void OnDictionaryIdDropdownValueChanged(int result) { - if ((int)dictionaryId != result) { + if ((int)dictionaryId != result) + { dictionaryId = (ArUcoDictionary)result; - dictionary = Aruco.getPredefinedDictionary ((int)dictionaryId); + dictionary = Aruco.getPredefinedDictionary((int)dictionaryId); - if (isImagesInputMode) { - InitializeImagesInputMode (); - } else { - if (webCamTextureToMatHelper.IsInitialized ()) - webCamTextureToMatHelper.Initialize (); + if (isImagesInputMode) + { + InitializeImagesInputMode(); + } + else + { + if (webCamTextureToMatHelper.IsInitialized()) + webCamTextureToMatHelper.Initialize(); } } } @@ -990,14 +1048,18 @@ public void OnDictionaryIdDropdownValueChanged(int result) /// public void OnSquaresXDropdownValueChanged(int result) { - if ((int)squaresX != result + 1) { + if ((int)squaresX != result + 1) + { squaresX = (NumberOfSquaresX)(result + 1); - if (isImagesInputMode) { - InitializeImagesInputMode (); - } else { - if (webCamTextureToMatHelper.IsInitialized ()) - webCamTextureToMatHelper.Initialize (); + if (isImagesInputMode) + { + InitializeImagesInputMode(); + } + else + { + if (webCamTextureToMatHelper.IsInitialized()) + webCamTextureToMatHelper.Initialize(); } } } @@ -1007,14 +1069,18 @@ public void OnSquaresXDropdownValueChanged(int result) /// public void OnSquaresYDropdownValueChanged(int result) { - if ((int)squaresY != result + 1) { + if ((int)squaresY != result + 1) + { squaresY = (NumberOfSquaresY)(result + 1); - if (isImagesInputMode) { - InitializeImagesInputMode (); - } else { - if (webCamTextureToMatHelper.IsInitialized ()) - webCamTextureToMatHelper.Initialize (); + if (isImagesInputMode) + { + InitializeImagesInputMode(); + } + else + { + if (webCamTextureToMatHelper.IsInitialized()) + webCamTextureToMatHelper.Initialize(); } } } @@ -1022,13 +1088,16 @@ public void OnSquaresYDropdownValueChanged(int result) /// /// Raises the capture button click event. /// - public void OnCaptureButtonClick () + public void OnCaptureButtonClick() { - if (isImagesInputMode) { + if (isImagesInputMode) + { if (!isCalibrating) - InitializeImagesInputMode (); - StartCoroutine ("CalibrateCameraUsingImages"); - } else { + InitializeImagesInputMode(); + StartCoroutine("CalibrateCameraUsingImages"); + } + else + { shouldCaptureFrame = true; } } @@ -1036,110 +1105,99 @@ public void OnCaptureButtonClick () /// /// Raises the reset button click event. /// - public void OnResetButtonClick () + public void OnResetButtonClick() { - if (isImagesInputMode) { + if (isImagesInputMode) + { if (!isCalibrating) - InitializeImagesInputMode (); - } else { - ResetCalibration (); + InitializeImagesInputMode(); + } + else + { + ResetCalibration(); } } /// /// Raises the save button click event. /// - public void OnSaveButtonClick () + public void OnSaveButtonClick() { - string saveDirectoryPath = Path.Combine (Application.persistentDataPath, "HoloLensArUcoCameraCalibrationExample"); + string saveDirectoryPath = Path.Combine(Application.persistentDataPath, "HoloLensArUcoCameraCalibrationExample"); - if (!Directory.Exists (saveDirectoryPath)) { - Directory.CreateDirectory (saveDirectoryPath); + if (!Directory.Exists(saveDirectoryPath)) + { + Directory.CreateDirectory(saveDirectoryPath); } - string calibratonDirectoryName = "camera_parameters" + bgrMat.width () + "x" + bgrMat.height (); - string saveCalibratonFileDirectoryPath = Path.Combine (saveDirectoryPath, calibratonDirectoryName); + string calibratonDirectoryName = "camera_parameters" + bgrMat.width() + "x" + bgrMat.height(); + string saveCalibratonFileDirectoryPath = Path.Combine(saveDirectoryPath, calibratonDirectoryName); // Clean up old files. - if (Directory.Exists (saveCalibratonFileDirectoryPath)) { - DirectoryInfo directoryInfo = new DirectoryInfo (saveCalibratonFileDirectoryPath); - foreach (FileInfo fileInfo in directoryInfo.GetFiles()) { - if ((fileInfo.Attributes & FileAttributes.ReadOnly) == FileAttributes.ReadOnly) { + if (Directory.Exists(saveCalibratonFileDirectoryPath)) + { + DirectoryInfo directoryInfo = new DirectoryInfo(saveCalibratonFileDirectoryPath); + foreach (FileInfo fileInfo in directoryInfo.GetFiles()) + { + if ((fileInfo.Attributes & FileAttributes.ReadOnly) == FileAttributes.ReadOnly) + { fileInfo.Attributes = FileAttributes.Normal; } } - if ((directoryInfo.Attributes & FileAttributes.ReadOnly) == FileAttributes.ReadOnly) { + if ((directoryInfo.Attributes & FileAttributes.ReadOnly) == FileAttributes.ReadOnly) + { directoryInfo.Attributes = FileAttributes.Directory; } directoryInfo.Delete(true); } - Directory.CreateDirectory (saveCalibratonFileDirectoryPath); + Directory.CreateDirectory(saveCalibratonFileDirectoryPath); // save the calibraton file. - string savePath = Path.Combine (saveCalibratonFileDirectoryPath, calibratonDirectoryName + ".xml"); + string savePath = Path.Combine(saveCalibratonFileDirectoryPath, calibratonDirectoryName + ".xml"); int frameCount = (markerType == MarkerType.ChArUcoBoard) ? allCorners.Count : imagePoints.Count; CameraParameters param = new CameraParameters(frameCount, bgrMat.width(), bgrMat.height(), calibrationFlags, camMatrix, distCoeffs, repErr); - XmlSerializer serializer = new XmlSerializer( typeof( CameraParameters ) ); - using (var stream = new FileStream (savePath, FileMode.Create)) { - serializer.Serialize (stream, param); + XmlSerializer serializer = new XmlSerializer(typeof(CameraParameters)); + using (var stream = new FileStream(savePath, FileMode.Create)) + { + serializer.Serialize(stream, param); } // save the calibration images. - #if UNITY_WEBGL && !UNITY_EDITOR +#if UNITY_WEBGL && !UNITY_EDITOR string format = "jpg"; MatOfInt compressionParams = new MatOfInt(Imgcodecs.IMWRITE_JPEG_QUALITY, 100); - #else +#else string format = "png"; MatOfInt compressionParams = new MatOfInt(Imgcodecs.IMWRITE_PNG_COMPRESSION, 0); - #endif - for (int i = 0; i < allImgs.Count; ++i) { - Imgcodecs.imwrite (Path.Combine (saveCalibratonFileDirectoryPath, calibratonDirectoryName + "_" + i.ToString("00") + "." + format), allImgs[i], compressionParams); +#endif + for (int i = 0; i < allImgs.Count; ++i) + { + Imgcodecs.imwrite(Path.Combine(saveCalibratonFileDirectoryPath, calibratonDirectoryName + "_" + i.ToString("00") + "." + format), allImgs[i], compressionParams); } savePathInputField.text = savePath; - Debug.Log ("Saved the CameraParameters to disk in XML file format."); - Debug.Log ("savePath: " + savePath); - } - - private void SetupGestureRecognizer () - { - m_GestureRecognizer = new GestureRecognizer (); - m_GestureRecognizer.SetRecognizableGestures (GestureSettings.Tap); - #if UNITY_2017_2_OR_NEWER - m_GestureRecognizer.Tapped += OnTappedEvent; - #else - m_GestureRecognizer.TappedEvent += OnTappedEvent; - #endif - m_GestureRecognizer.StartCapturingGestures (); + Debug.Log("Saved the CameraParameters to disk in XML file format."); + Debug.Log("savePath: " + savePath); } - private void ReleaseGestureRecognizer () + /// + /// Raises the tapped event. + /// + public void OnTapped(MixedRealityPointerEventData eventData) { - if (m_GestureRecognizer != null && m_GestureRecognizer.IsCapturingGestures()) { - m_GestureRecognizer.StopCapturingGestures (); - #if UNITY_2017_2_OR_NEWER - m_GestureRecognizer.Tapped -= OnTappedEvent; - #else - m_GestureRecognizer.TappedEvent -= OnTappedEvent; - #endif - m_GestureRecognizer.Dispose (); - } - } + Debug.Log("OnTapped!"); - #if UNITY_2017_2_OR_NEWER - private void OnTappedEvent (TappedEventArgs args) - #else - private void OnTappedEvent (InteractionSourceKind source, int tapCount, Ray headRay) - #endif - { // Determine if a Gaze pointer is over a GUI. - if (GazeManager.Instance.HitObject != null && GazeManager.Instance.HitObject.transform.name == "Text") { + if (eventData.selectedObject != null && (eventData.selectedObject.GetComponent