diff --git a/Assets/HoloLensWithOpenCVForUnityExample/HoloLensArUcoExample.meta b/Assets/HoloLensWithOpenCVForUnityExample/HLArUcoExample.meta similarity index 100% rename from Assets/HoloLensWithOpenCVForUnityExample/HoloLensArUcoExample.meta rename to Assets/HoloLensWithOpenCVForUnityExample/HLArUcoExample.meta diff --git a/Assets/HoloLensWithOpenCVForUnityExample/HoloLensArUcoExample/CameraParameters.cs b/Assets/HoloLensWithOpenCVForUnityExample/HLArUcoExample/CameraParameters.cs similarity index 96% rename from Assets/HoloLensWithOpenCVForUnityExample/HoloLensArUcoExample/CameraParameters.cs rename to Assets/HoloLensWithOpenCVForUnityExample/HLArUcoExample/CameraParameters.cs index 4efe299..f0b69c2 100644 --- a/Assets/HoloLensWithOpenCVForUnityExample/HoloLensArUcoExample/CameraParameters.cs +++ b/Assets/HoloLensWithOpenCVForUnityExample/HLArUcoExample/CameraParameters.cs @@ -1,62 +1,62 @@ -using System; -using OpenCVForUnity.CoreModule; - -namespace HoloLensWithOpenCVForUnityExample -{ - [System.Serializable] - public struct CameraParameters - { - public string calibration_date; - public int frames_count; - public int image_width; - public int image_height; - public int calibration_flags; - public double[] camera_matrix; - public double[] distortion_coefficients; - public double avg_reprojection_error; - - public CameraParameters(int frames_count, int image_width, int image_height, int calibration_flags, double[] camera_matrix, double[] distortion_coefficients, double avg_reprojection_error) - { - this.calibration_date = DateTime.Now.ToString(); - this.frames_count = frames_count; - this.image_width = image_width; - this.image_height = image_height; - this.calibration_flags = calibration_flags; - this.camera_matrix = camera_matrix; - this.distortion_coefficients = distortion_coefficients; - this.avg_reprojection_error = avg_reprojection_error; - } - - public CameraParameters(int frames_count, int image_width, int image_height, int calibration_flags, Mat camera_matrix, Mat distortion_coefficients, double avg_reprojection_error) - { - double[] camera_matrixArr = new double[camera_matrix.total()]; - camera_matrix.get(0, 0, camera_matrixArr); - - double[] distortion_coefficientsArr = new double[distortion_coefficients.total()]; - distortion_coefficients.get(0, 0, distortion_coefficientsArr); - - this.calibration_date = DateTime.Now.ToString(); - this.frames_count = frames_count; - this.image_width = image_width; - this.image_height = image_height; - this.calibration_flags = calibration_flags; - this.camera_matrix = camera_matrixArr; - this.distortion_coefficients = distortion_coefficientsArr; - this.avg_reprojection_error = avg_reprojection_error; - } - - public Mat GetCameraMatrix() - { - Mat m = new Mat(3, 3, CvType.CV_64FC1); - m.put(0, 0, camera_matrix); - return m; - } - - public Mat GetDistortionCoefficients() - { - Mat m = new Mat(distortion_coefficients.Length, 1, CvType.CV_64FC1); - m.put(0, 0, distortion_coefficients); - return m; - } - } -} +using System; +using OpenCVForUnity.CoreModule; + +namespace HoloLensWithOpenCVForUnityExample +{ + [System.Serializable] + public struct CameraParameters + { + public string calibration_date; + public int frames_count; + public int image_width; + public int image_height; + public int calibration_flags; + public double[] camera_matrix; + public double[] distortion_coefficients; + public double avg_reprojection_error; + + public CameraParameters(int frames_count, int image_width, int image_height, int calibration_flags, double[] camera_matrix, double[] distortion_coefficients, double avg_reprojection_error) + { + this.calibration_date = DateTime.Now.ToString(); + this.frames_count = frames_count; + this.image_width = image_width; + this.image_height = image_height; + this.calibration_flags = calibration_flags; + this.camera_matrix = camera_matrix; + this.distortion_coefficients = distortion_coefficients; + this.avg_reprojection_error = avg_reprojection_error; + } + + public CameraParameters(int frames_count, int image_width, int image_height, int calibration_flags, Mat camera_matrix, Mat distortion_coefficients, double avg_reprojection_error) + { + double[] camera_matrixArr = new double[camera_matrix.total()]; + camera_matrix.get(0, 0, camera_matrixArr); + + double[] distortion_coefficientsArr = new double[distortion_coefficients.total()]; + distortion_coefficients.get(0, 0, distortion_coefficientsArr); + + this.calibration_date = DateTime.Now.ToString(); + this.frames_count = frames_count; + this.image_width = image_width; + this.image_height = image_height; + this.calibration_flags = calibration_flags; + this.camera_matrix = camera_matrixArr; + this.distortion_coefficients = distortion_coefficientsArr; + this.avg_reprojection_error = avg_reprojection_error; + } + + public Mat GetCameraMatrix() + { + Mat m = new Mat(3, 3, CvType.CV_64FC1); + m.put(0, 0, camera_matrix); + return m; + } + + public Mat GetDistortionCoefficients() + { + Mat m = new Mat(distortion_coefficients.Length, 1, CvType.CV_64FC1); + m.put(0, 0, distortion_coefficients); + return m; + } + } +} diff --git a/Assets/HoloLensWithOpenCVForUnityExample/HoloLensArUcoExample/CameraParameters.cs.meta b/Assets/HoloLensWithOpenCVForUnityExample/HLArUcoExample/CameraParameters.cs.meta similarity index 100% rename from Assets/HoloLensWithOpenCVForUnityExample/HoloLensArUcoExample/CameraParameters.cs.meta rename to Assets/HoloLensWithOpenCVForUnityExample/HLArUcoExample/CameraParameters.cs.meta diff --git a/Assets/HoloLensWithOpenCVForUnityExample/HoloLensArUcoExample/HoloLensArUcoCameraCalibrationExample.cs b/Assets/HoloLensWithOpenCVForUnityExample/HLArUcoExample/HLArUcoCameraCalibrationExample.cs similarity index 96% rename from Assets/HoloLensWithOpenCVForUnityExample/HoloLensArUcoExample/HoloLensArUcoCameraCalibrationExample.cs rename to Assets/HoloLensWithOpenCVForUnityExample/HLArUcoExample/HLArUcoCameraCalibrationExample.cs index 1359ce8..388dc9b 100644 --- a/Assets/HoloLensWithOpenCVForUnityExample/HoloLensArUcoExample/HoloLensArUcoCameraCalibrationExample.cs +++ b/Assets/HoloLensWithOpenCVForUnityExample/HLArUcoExample/HLArUcoCameraCalibrationExample.cs @@ -1,1256 +1,1262 @@ -using UnityEngine; -using UnityEngine.UI; -using UnityEngine.SceneManagement; -using System; -using System.Collections; -using System.Collections.Generic; -using System.Xml.Serialization; -using System.IO; -using System.Linq; -using OpenCVForUnity.CoreModule; -using OpenCVForUnity.ArucoModule; -using OpenCVForUnity.UnityUtils.Helper; -using OpenCVForUnity.ImgprocModule; -using OpenCVForUnity.UnityUtils; -using OpenCVForUnity.Calib3dModule; -using OpenCVForUnity.ImgcodecsModule; -using HoloLensWithOpenCVForUnity.UnityUtils.Helper; -using Microsoft.MixedReality.Toolkit.Input; - -namespace HoloLensWithOpenCVForUnityExample -{ - /// - /// HoloLens ArUco Camera Calibration Example - /// An example of camera calibration using the aruco module. - /// Referring to https://github.com/opencv/opencv_contrib/blob/master/modules/aruco/samples/calibrate_camera.cpp. - /// https://github.com/opencv/opencv/blob/master/samples/cpp/tutorial_code/calib3d/camera_calibration/camera_calibration.cpp - /// https://docs.opencv.org/3.2.0/da/d13/tutorial_aruco_calibration.html - /// https://docs.opencv.org/3.4.0/d7/d21/tutorial_interactive_calibration.html - /// - [RequireComponent(typeof(HololensCameraStreamToMatHelper))] - public class HoloLensArUcoCameraCalibrationExample : MonoBehaviour - { - /// - /// The preview quad. - /// - public GameObject previewQuad; - - /// - /// The marker type. - /// - public MarkerType markerType = MarkerType.ChArUcoBoard; - - /// - /// The marker type dropdown. - /// - public Dropdown markerTypeDropdown; - - /// - /// The dictionary identifier. - /// - public ArUcoDictionary dictionaryId = ArUcoDictionary.DICT_6X6_250; - - /// - /// The dictionary id dropdown. - /// - public Dropdown dictionaryIdDropdown; - - /// - /// Number of squares in X direction. - /// - public NumberOfSquaresX squaresX = NumberOfSquaresX.X_5; - - /// - /// The squares X dropdown. - /// - public Dropdown squaresXDropdown; - - /// - /// Number of squares in X direction. - /// - public NumberOfSquaresY squaresY = NumberOfSquaresY.Y_7; - - /// - /// The squares X dropdown. - /// - public Dropdown squaresYDropdown; - - /// - /// The save path input field. - /// - public InputField savePathInputField; - - /// - /// Determines if refine marker detection. (only valid for ArUco boards) - /// - public bool refineMarkerDetection = true; - - - [Header("Extra Option")] - - /// - /// Determines if calibrates camera using the list of calibration images. - /// - [TooltipAttribute("Determines if calibrates camera using the list of calibration images.")] - public bool isImagesInputMode = false; - - /// - /// The calibration images directory path. - /// Set a relative directory path from the starting point of the "StreamingAssets" folder. e.g. "calibration_images/". - /// - [TooltipAttribute("Set a relative directory path from the starting point of the \"StreamingAssets\" folder. e.g. \"calibration_images\"")] - public string calibrationImagesDirectory = "calibration_images"; - - /// - /// The texture. - /// - Texture2D texture; - - /// - /// The webcam texture to mat helper. - /// - HololensCameraStreamToMatHelper webCamTextureToMatHelper; - - /// - /// The bgr mat. - /// - Mat bgrMat; - - /// - /// The rgba mat. - /// - Mat rgbaMat; - - /// - /// The cameraparam matrix. - /// - Mat camMatrix; - - /// - /// The distortion coeffs. - /// - MatOfDouble distCoeffs; - - /// - /// The identifiers. - /// - Mat ids; - - /// - /// The corners. - /// - List corners; - - /// - /// The rejected corners. - /// - List rejectedCorners; - - /// - /// The rvecs. - /// - List rvecs; - - /// - /// The tvecs. - /// - List tvecs; - - /// - /// The detector parameters. - /// - DetectorParameters detectorParams; - - /// - /// The dictionary. - /// - Dictionary dictionary; - - /// - /// The recovered identifiers. - /// - Mat recoveredIdxs; - - const int calibrationFlags = 0;// Calib3d.CALIB_FIX_K3 | Calib3d.CALIB_FIX_K4 | Calib3d.CALIB_FIX_K5 - double repErr = 0; - bool shouldCaptureFrame = false; - - // for ChArUcoBoard. - // chessboard square side length (normally in meters) - const float chArUcoBoradSquareLength = 0.04f; - // marker side length (same unit than squareLength) - const float chArUcoBoradMarkerLength = 0.02f; - const int charucoMinMarkers = 2; - Mat charucoCorners; - Mat charucoIds; - CharucoBoard charucoBoard; - List> allCorners; - List allIds; - List allImgs; - - // for OthearMarkers. - // square size in some user-defined units (1 by default) - const float squareSize = 1f; - List imagePoints; - bool isInitialized = false; - bool isCalibrating = false; - - // Use this for initialization - IEnumerator Start() - { - webCamTextureToMatHelper = gameObject.GetComponent(); - webCamTextureToMatHelper.outputColorFormat = WebCamTextureToMatHelper.ColorFormat.GRAY; - - // fix the screen orientation. - Screen.orientation = ScreenOrientation.LandscapeLeft; - // wait for the screen orientation to change. - yield return null; - - if (markerTypeDropdown.value != (int)markerType || dictionaryIdDropdown.value != (int)dictionaryId - || squaresXDropdown.value != (int)squaresX - 1 || squaresYDropdown.value != (int)squaresY - 1) - { - markerTypeDropdown.value = (int)markerType; - dictionaryIdDropdown.value = (int)dictionaryId; - squaresXDropdown.value = (int)squaresX - 1; - squaresYDropdown.value = (int)squaresY - 1; - } - dictionaryIdDropdown.interactable = (markerType == MarkerType.ChArUcoBoard); - -#if UNITY_WEBGL && !UNITY_EDITOR - isImagesInputMode = false; -#endif - if (isImagesInputMode) - { - isImagesInputMode = InitializeImagesInputMode(); - } - - if (!isImagesInputMode) - { - webCamTextureToMatHelper.Initialize(); - } - } - - /// - /// Raises the webcam texture to mat helper initialized event. - /// - public void OnWebCamTextureToMatHelperInitialized() - { - Debug.Log("OnWebCamTextureToMatHelperInitialized"); - - Mat webCamTextureMat = webCamTextureToMatHelper.GetMat(); - - InitializeCalibraton(webCamTextureMat); - - // if WebCamera is frontFaceing, flip Mat. - webCamTextureToMatHelper.flipHorizontal = webCamTextureToMatHelper.IsFrontFacing(); - } - - /// - /// Raises the webcam texture to mat helper disposed event. - /// - public void OnWebCamTextureToMatHelperDisposed() - { - Debug.Log("OnWebCamTextureToMatHelperDisposed"); - - DisposeCalibraton(); - } - - /// - /// Raises the webcam texture to mat helper error occurred event. - /// - /// Error code. - public void OnWebCamTextureToMatHelperErrorOccurred(WebCamTextureToMatHelper.ErrorCode errorCode) - { - Debug.Log("OnWebCamTextureToMatHelperErrorOccurred " + errorCode); - } - - // Update is called once per frame - void Update() - { - if (isImagesInputMode) - return; - - if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame()) - { - - Mat grayMat = webCamTextureToMatHelper.GetMat(); - - if (shouldCaptureFrame) - { - shouldCaptureFrame = false; - Mat frameMat = grayMat.clone(); - double e = CaptureFrame(frameMat); - if (e > 0) - repErr = e; - } - - DrawFrame(grayMat, bgrMat); - Imgproc.cvtColor(bgrMat, rgbaMat, Imgproc.COLOR_BGR2RGBA); - - Utils.fastMatToTexture2D(rgbaMat, texture); - } - } - - private void InitializeCalibraton(Mat frameMat) - { - Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation); - - float width = frameMat.width(); - float height = frameMat.height(); - - texture = new Texture2D(frameMat.cols(), frameMat.rows(), TextureFormat.RGBA32, false); - texture.wrapMode = TextureWrapMode.Clamp; - - previewQuad.GetComponent().material.mainTexture = texture; - previewQuad.transform.localScale = new Vector3(0.2f * width / height, 0.2f, 1); - - float imageSizeScale = 1.0f; - - // set cameraparam. - camMatrix = CreateCameraMatrix(width, height); - Debug.Log("camMatrix " + camMatrix.dump()); - - distCoeffs = new MatOfDouble(0, 0, 0, 0, 0); - Debug.Log("distCoeffs " + distCoeffs.dump()); - - // calibration camera. - Size imageSize = new Size(width * imageSizeScale, height * imageSizeScale); - double apertureWidth = 0; - double apertureHeight = 0; - double[] fovx = new double[1]; - double[] fovy = new double[1]; - double[] focalLength = new double[1]; - Point principalPoint = new Point(0, 0); - double[] aspectratio = new double[1]; - - Calib3d.calibrationMatrixValues(camMatrix, imageSize, apertureWidth, apertureHeight, fovx, fovy, focalLength, principalPoint, aspectratio); - - Debug.Log("imageSize " + imageSize.ToString()); - Debug.Log("apertureWidth " + apertureWidth); - Debug.Log("apertureHeight " + apertureHeight); - Debug.Log("fovx " + fovx[0]); - Debug.Log("fovy " + fovy[0]); - Debug.Log("focalLength " + focalLength[0]); - Debug.Log("principalPoint " + principalPoint.ToString()); - Debug.Log("aspectratio " + aspectratio[0]); - - - bgrMat = new Mat(frameMat.rows(), frameMat.cols(), CvType.CV_8UC3); - rgbaMat = new Mat(frameMat.rows(), frameMat.cols(), CvType.CV_8UC4); - ids = new Mat(); - corners = new List(); - rejectedCorners = new List(); - rvecs = new List(); - tvecs = new List(); - - detectorParams = DetectorParameters.create(); - detectorParams.set_cornerRefinementMethod(1);// do cornerSubPix() of OpenCV. - dictionary = Aruco.getPredefinedDictionary((int)dictionaryId); - - recoveredIdxs = new Mat(); - - charucoCorners = new Mat(); - charucoIds = new Mat(); - charucoBoard = CharucoBoard.create((int)squaresX, (int)squaresY, chArUcoBoradSquareLength, chArUcoBoradMarkerLength, dictionary); - - - allCorners = new List>(); - allIds = new List(); - allImgs = new List(); - - imagePoints = new List(); - - isInitialized = true; - } - - private void DisposeCalibraton() - { - ResetCalibration(); - - if (bgrMat != null) - bgrMat.Dispose(); - if (rgbaMat != null) - rgbaMat.Dispose(); - if (ids != null) - ids.Dispose(); - foreach (var item in corners) - { - item.Dispose(); - } - corners.Clear(); - foreach (var item in rejectedCorners) - { - item.Dispose(); - } - rejectedCorners.Clear(); - foreach (var item in rvecs) - { - item.Dispose(); - } - rvecs.Clear(); - foreach (var item in tvecs) - { - item.Dispose(); - } - tvecs.Clear(); - - if (recoveredIdxs != null) - recoveredIdxs.Dispose(); - - if (charucoCorners != null) - charucoCorners.Dispose(); - if (charucoIds != null) - charucoIds.Dispose(); - if (charucoBoard != null) - charucoBoard.Dispose(); - - isInitialized = false; - } - - private void DrawFrame(Mat grayMat, Mat bgrMat) - { - Imgproc.cvtColor(grayMat, bgrMat, Imgproc.COLOR_GRAY2BGR); - - switch (markerType) - { - default: - case MarkerType.ChArUcoBoard: - // detect markers. - Aruco.detectMarkers(grayMat, dictionary, corners, ids, detectorParams, rejectedCorners, camMatrix, distCoeffs); - - // refine marker detection. - if (refineMarkerDetection) - { - Aruco.refineDetectedMarkers(grayMat, charucoBoard, corners, ids, rejectedCorners, camMatrix, distCoeffs, 10f, 3f, true, recoveredIdxs, detectorParams); - } - - // if at least one marker detected - if (ids.total() > 0) - { - Aruco.interpolateCornersCharuco(corners, ids, grayMat, charucoBoard, charucoCorners, charucoIds, camMatrix, distCoeffs, charucoMinMarkers); - - // draw markers. - Aruco.drawDetectedMarkers(bgrMat, corners, ids, new Scalar(0, 255, 0, 255)); - // if at least one charuco corner detected - if (charucoIds.total() > 0) - { - Aruco.drawDetectedCornersCharuco(bgrMat, charucoCorners, charucoIds, new Scalar(0, 0, 255, 255)); - } - } - break; - case MarkerType.ChessBoard: - case MarkerType.CirclesGlid: - case MarkerType.AsymmetricCirclesGlid: - // detect markers. - MatOfPoint2f points = new MatOfPoint2f(); - bool found = false; - - switch (markerType) - { - default: - case MarkerType.ChessBoard: - found = Calib3d.findChessboardCorners(grayMat, new Size((int)squaresX, (int)squaresY), points, Calib3d.CALIB_CB_ADAPTIVE_THRESH | Calib3d.CALIB_CB_FAST_CHECK | Calib3d.CALIB_CB_NORMALIZE_IMAGE); - break; - case MarkerType.CirclesGlid: - found = Calib3d.findCirclesGrid(grayMat, new Size((int)squaresX, (int)squaresY), points, Calib3d.CALIB_CB_SYMMETRIC_GRID); - break; - case MarkerType.AsymmetricCirclesGlid: - found = Calib3d.findCirclesGrid(grayMat, new Size((int)squaresX, (int)squaresY), points, Calib3d.CALIB_CB_ASYMMETRIC_GRID); - break; - } - - if (found) - { - if (markerType == MarkerType.ChessBoard) - Imgproc.cornerSubPix(grayMat, points, new Size(5, 5), new Size(-1, -1), new TermCriteria(TermCriteria.EPS + TermCriteria.COUNT, 30, 0.1)); - - // draw markers. - Calib3d.drawChessboardCorners(bgrMat, new Size((int)squaresX, (int)squaresY), points, found); - } - break; - } - - double[] camMatrixArr = new double[(int)camMatrix.total()]; - camMatrix.get(0, 0, camMatrixArr); - double[] distCoeffsArr = new double[(int)distCoeffs.total()]; - distCoeffs.get(0, 0, distCoeffsArr); - - int ff = Imgproc.FONT_HERSHEY_SIMPLEX; - double fs = 0.4; - Scalar c = new Scalar(255, 255, 255, 255); - int t = 0; - int lt = Imgproc.LINE_AA; - bool blo = false; - int frameCount = (markerType == MarkerType.ChArUcoBoard) ? allCorners.Count : imagePoints.Count; - Imgproc.putText(bgrMat, frameCount + " FRAME CAPTURED", new Point(bgrMat.cols() - 300, 20), ff, fs, c, t, lt, blo); - Imgproc.putText(bgrMat, "IMAGE_WIDTH: " + bgrMat.width(), new Point(bgrMat.cols() - 300, 40), ff, fs, c, t, lt, blo); - Imgproc.putText(bgrMat, "IMAGE_HEIGHT: " + bgrMat.height(), new Point(bgrMat.cols() - 300, 60), ff, fs, c, t, lt, blo); - Imgproc.putText(bgrMat, "CALIBRATION_FLAGS: " + calibrationFlags, new Point(bgrMat.cols() - 300, 80), ff, fs, c, t, lt, blo); - - Imgproc.putText(bgrMat, "CAMERA_MATRIX: ", new Point(bgrMat.cols() - 300, 100), ff, fs, c, t, lt, blo); - for (int i = 0; i < camMatrixArr.Length; i = i + 3) - { - Imgproc.putText(bgrMat, " " + camMatrixArr[i] + ", " + camMatrixArr[i + 1] + ", " + camMatrixArr[i + 2] + ",", new Point(bgrMat.cols() - 300, 120 + 20 * i / 3), ff, fs, c, t, lt, blo); - } - Imgproc.putText(bgrMat, "DISTORTION_COEFFICIENTS: ", new Point(bgrMat.cols() - 300, 180), ff, fs, c, t, lt, blo); - for (int i = 0; i < distCoeffsArr.Length; ++i) - { - Imgproc.putText(bgrMat, " " + distCoeffsArr[i] + ",", new Point(bgrMat.cols() - 300, 200 + 20 * i), ff, fs, c, t, lt, blo); - } - Imgproc.putText(bgrMat, "AVG_REPROJECTION_ERROR: " + repErr, new Point(bgrMat.cols() - 300, 300), ff, fs, c, t, lt, blo); - - if (frameCount == 0) - Imgproc.putText(bgrMat, "To calibration start, please press the calibration button or do air tap gesture!", new Point(5, bgrMat.rows() - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(255, 255, 255, 255), 1, Imgproc.LINE_AA, false); - } - - private double CaptureFrame(Mat frameMat) - { - double repErr = -1; - - switch (markerType) - { - default: - case MarkerType.ChArUcoBoard: - List corners = new List(); - Mat ids = new Mat(); - Aruco.detectMarkers(frameMat, dictionary, corners, ids, detectorParams, rejectedCorners, camMatrix, distCoeffs); - - if (refineMarkerDetection) - { - Aruco.refineDetectedMarkers(frameMat, charucoBoard, corners, ids, rejectedCorners, camMatrix, distCoeffs, 10f, 3f, true, recoveredIdxs, detectorParams); - } - - if (ids.total() > 0) - { - Debug.Log("Frame captured."); - - allCorners.Add(corners); - allIds.Add(ids); - allImgs.Add(frameMat); - } - else - { - - Debug.Log("Invalid frame."); - - frameMat.Dispose(); - if (ids != null) - ids.Dispose(); - foreach (var item in corners) - { - item.Dispose(); - } - corners.Clear(); - - return -1; - } - - // calibrate camera using aruco markers - //double arucoRepErr = CalibrateCameraAruco (allCorners, allIds, charucoBoard, frameMat.size(), camMatrix, distCoeffs, rvecs, tvecs, calibrationFlags); - //Debug.Log ("arucoRepErr: " + arucoRepErr); - - // calibrate camera using charuco - repErr = CalibrateCameraCharuco(allCorners, allIds, charucoBoard, frameMat.size(), camMatrix, distCoeffs, rvecs, tvecs, calibrationFlags, calibrationFlags); - - break; - case MarkerType.ChessBoard: - case MarkerType.CirclesGlid: - case MarkerType.AsymmetricCirclesGlid: - - MatOfPoint2f points = new MatOfPoint2f(); - Size patternSize = new Size((int)squaresX, (int)squaresY); - - bool found = false; - switch (markerType) - { - default: - case MarkerType.ChessBoard: - found = Calib3d.findChessboardCorners(frameMat, patternSize, points, Calib3d.CALIB_CB_ADAPTIVE_THRESH | Calib3d.CALIB_CB_FAST_CHECK | Calib3d.CALIB_CB_NORMALIZE_IMAGE); - break; - case MarkerType.CirclesGlid: - found = Calib3d.findCirclesGrid(frameMat, patternSize, points, Calib3d.CALIB_CB_SYMMETRIC_GRID); - break; - case MarkerType.AsymmetricCirclesGlid: - found = Calib3d.findCirclesGrid(frameMat, patternSize, points, Calib3d.CALIB_CB_ASYMMETRIC_GRID); - break; - } - - if (found) - { - Debug.Log("Frame captured."); - if (markerType == MarkerType.ChessBoard) - Imgproc.cornerSubPix(frameMat, points, new Size(5, 5), new Size(-1, -1), new TermCriteria(TermCriteria.EPS + TermCriteria.COUNT, 30, 0.1)); - - imagePoints.Add(points); - allImgs.Add(frameMat); - } - else - { - Debug.Log("Invalid frame."); - frameMat.Dispose(); - if (points != null) - points.Dispose(); - return -1; - } - - if (imagePoints.Count < 1) - { - Debug.Log("Not enough points for calibration."); - repErr = -1; - } - else - { - - MatOfPoint3f objectPoint = new MatOfPoint3f(new Mat(imagePoints[0].rows(), 1, CvType.CV_32FC3)); - CalcChessboardCorners(patternSize, squareSize, objectPoint, markerType); - - List objectPoints = new List(); - for (int i = 0; i < imagePoints.Count; ++i) - { - objectPoints.Add(objectPoint); - } - - repErr = Calib3d.calibrateCamera(objectPoints, imagePoints, frameMat.size(), camMatrix, distCoeffs, rvecs, tvecs, calibrationFlags); - objectPoint.Dispose(); - } - - break; - } - - Debug.Log("repErr: " + repErr); - Debug.Log("camMatrix: " + camMatrix.dump()); - Debug.Log("distCoeffs: " + distCoeffs.dump()); - - return repErr; - } - - private double CalibrateCameraAruco(List> allCorners, List allIds, CharucoBoard board, Size imageSize, Mat cameraMatrix, Mat distCoeffs, List rvecs = null, List tvecs = null, int calibrationFlags = 0) - { - // prepare data for calibration - int nFrames = allCorners.Count; - int allLen = 0; - int[] markerCounterPerFrameArr = new int[allCorners.Count]; - for (int i = 0; i < nFrames; ++i) - { - markerCounterPerFrameArr[i] = allCorners[i].Count; - allLen += allCorners[i].Count; - } - - int[] allIdsConcatenatedArr = new int[allLen]; - int index = 0; - for (int j = 0; j < allIds.Count; ++j) - { - int[] idsArr = new int[(int)allIds[j].total()]; - allIds[j].get(0, 0, idsArr); - - for (int k = 0; k < idsArr.Length; ++k) - { - allIdsConcatenatedArr[index + k] = (int)idsArr[k]; - } - index += idsArr.Length; - } - - using (Mat allIdsConcatenated = new Mat(1, allLen, CvType.CV_32SC1)) - using (Mat markerCounterPerFrame = new Mat(1, nFrames, CvType.CV_32SC1)) - { - - List allCornersConcatenated = new List(); - foreach (var c in allCorners) - { - foreach (var m in c) - { - allCornersConcatenated.Add(m); - } - } - - allIdsConcatenated.put(0, 0, allIdsConcatenatedArr); - markerCounterPerFrame.put(0, 0, markerCounterPerFrameArr); - - if (rvecs == null) rvecs = new List(); - if (tvecs == null) tvecs = new List(); - - return Aruco.calibrateCameraAruco(allCornersConcatenated, allIdsConcatenated, markerCounterPerFrame, board, imageSize, cameraMatrix, distCoeffs, rvecs, tvecs, calibrationFlags); - } - } - - private double CalibrateCameraCharuco(List> allCorners, List allIds, CharucoBoard board, Size imageSize, Mat cameraMatrix, Mat distCoeffs, List rvecs = null, List tvecs = null, int calibrationFlags = 0, int minMarkers = 2) - { - // prepare data for charuco calibration - int nFrames = allCorners.Count; - List allCharucoCorners = new List(); - List allCharucoIds = new List(); - List filteredImages = new List(); - - for (int i = 0; i < nFrames; ++i) - { - // interpolate using camera parameters - Mat currentCharucoCorners = new Mat(); - Mat currentCharucoIds = new Mat(); - - Aruco.interpolateCornersCharuco(allCorners[i], allIds[i], allImgs[i], board, currentCharucoCorners, currentCharucoIds, cameraMatrix, distCoeffs, minMarkers); - - if (charucoIds.total() > 0) - { - allCharucoCorners.Add(currentCharucoCorners); - allCharucoIds.Add(currentCharucoIds); - filteredImages.Add(allImgs[i]); - } - else - { - currentCharucoCorners.Dispose(); - currentCharucoIds.Dispose(); - } - } - - if (allCharucoCorners.Count < 1) - { - Debug.Log("Not enough corners for calibration."); - return -1; - } - - if (rvecs == null) rvecs = new List(); - if (tvecs == null) tvecs = new List(); - - return Aruco.calibrateCameraCharuco(allCharucoCorners, allCharucoIds, board, imageSize, cameraMatrix, distCoeffs, rvecs, tvecs, calibrationFlags); - } - - private void ResetCalibration() - { - foreach (var corners in allCorners) - { - foreach (var item in corners) - { - item.Dispose(); - } - } - allCorners.Clear(); - - foreach (var item in allIds) - { - item.Dispose(); - } - allIds.Clear(); - - foreach (var item in allImgs) - { - item.Dispose(); - } - allImgs.Clear(); - - repErr = 0; - camMatrix = CreateCameraMatrix(bgrMat.width(), bgrMat.height()); - distCoeffs = new MatOfDouble(0, 0, 0, 0, 0); - - foreach (var item in imagePoints) - { - item.Dispose(); - } - imagePoints.Clear(); - } - - private Mat CreateCameraMatrix(float width, float height) - { - int max_d = (int)Mathf.Max(width, height); - double fx = max_d; - double fy = max_d; - double cx = width / 2.0f; - double cy = height / 2.0f; - - Mat camMatrix = new Mat(3, 3, CvType.CV_64FC1); - camMatrix.put(0, 0, fx); - camMatrix.put(0, 1, 0); - camMatrix.put(0, 2, cx); - camMatrix.put(1, 0, 0); - camMatrix.put(1, 1, fy); - camMatrix.put(1, 2, cy); - camMatrix.put(2, 0, 0); - camMatrix.put(2, 1, 0); - camMatrix.put(2, 2, 1.0f); - - return camMatrix; - } - - private void CalcChessboardCorners(Size patternSize, float squareSize, MatOfPoint3f corners, MarkerType markerType) - { - if ((int)(patternSize.width * patternSize.height) != corners.rows()) - { - Debug.Log("Invalid corners size."); - corners.create((int)(patternSize.width * patternSize.height), 1, CvType.CV_32FC3); - } - - const int cn = 3; - float[] cornersArr = new float[corners.rows() * cn]; - int width = (int)patternSize.width; - int height = (int)patternSize.height; - - switch (markerType) - { - default: - case MarkerType.ChessBoard: - case MarkerType.CirclesGlid: - for (int i = 0; i < height; ++i) - { - for (int j = 0; j < width; ++j) - { - cornersArr[(i * width * cn) + (j * cn)] = j * squareSize; - cornersArr[(i * width * cn) + (j * cn) + 1] = i * squareSize; - cornersArr[(i * width * cn) + (j * cn) + 2] = 0; - } - } - corners.put(0, 0, cornersArr); - - break; - case MarkerType.AsymmetricCirclesGlid: - for (int i = 0; i < height; ++i) - { - for (int j = 0; j < width; ++j) - { - cornersArr[(i * width * cn) + (j * cn)] = (2 * j + i % 2) * squareSize; - cornersArr[(i * width * cn) + (j * cn) + 1] = i * squareSize; - cornersArr[(i * width * cn) + (j * cn) + 2] = 0; - } - } - corners.put(0, 0, cornersArr); - - break; - } - } - - private bool InitializeImagesInputMode() - { - if (isInitialized) - DisposeCalibraton(); - - if (String.IsNullOrEmpty(calibrationImagesDirectory)) - { - Debug.LogWarning("When using the images input mode, please set a calibration images directory path."); - return false; - } - - string dirPath = Path.Combine(Application.streamingAssetsPath, calibrationImagesDirectory); - if (!Directory.Exists(dirPath)) - { - Debug.LogWarning("The directory does not exist."); - return false; - } - string[] imageFiles = GetImageFilesInDirectory(dirPath); - if (imageFiles.Length < 1) - { - Debug.LogWarning("The image file does not exist."); - return false; - } - - Uri rootPath = new Uri(Application.streamingAssetsPath + System.IO.Path.AltDirectorySeparatorChar); - Uri fullPath = new Uri(imageFiles[0]); - string relativePath = rootPath.MakeRelativeUri(fullPath).ToString(); - - using (Mat gray = Imgcodecs.imread(Utils.getFilePath(relativePath), Imgcodecs.IMREAD_GRAYSCALE)) - { - - if (gray.total() == 0) - { - Debug.LogWarning("Invalid image file."); - return false; - } - - using (Mat bgr = new Mat(gray.size(), CvType.CV_8UC3)) - using (Mat bgra = new Mat(gray.size(), CvType.CV_8UC4)) - { - InitializeCalibraton(gray); - - DrawFrame(gray, bgr); - Imgproc.cvtColor(bgr, bgra, Imgproc.COLOR_BGR2RGBA); - Utils.fastMatToTexture2D(bgra, texture); - } - } - return true; - } - - private IEnumerator CalibrateCameraUsingImages() - { - string dirPath = Path.Combine(Application.streamingAssetsPath, calibrationImagesDirectory); - string[] imageFiles = GetImageFilesInDirectory(dirPath); - if (imageFiles.Length < 1) - yield break; - - isCalibrating = true; - markerTypeDropdown.interactable = dictionaryIdDropdown.interactable = squaresXDropdown.interactable = squaresYDropdown.interactable = false; - - Uri rootPath = new Uri(Application.streamingAssetsPath + System.IO.Path.AltDirectorySeparatorChar); - - foreach (var path in imageFiles) - { - - Uri fullPath = new Uri(path); - string relativePath = rootPath.MakeRelativeUri(fullPath).ToString(); - - using (Mat gray = Imgcodecs.imread(Utils.getFilePath(relativePath), Imgcodecs.IMREAD_GRAYSCALE)) - { - - if (gray.width() != bgrMat.width() || gray.height() != bgrMat.height()) - continue; - - Mat frameMat = gray.clone(); - double e = CaptureFrame(frameMat); - if (e > 0) - repErr = e; - - DrawFrame(gray, bgrMat); - Imgproc.cvtColor(bgrMat, rgbaMat, Imgproc.COLOR_BGR2RGBA); - - Utils.matToTexture2D(rgbaMat, texture); - } - yield return new WaitForSeconds(0.5f); - } - - isCalibrating = false; - markerTypeDropdown.interactable = dictionaryIdDropdown.interactable = squaresXDropdown.interactable = squaresYDropdown.interactable = true; - } - - private string[] GetImageFilesInDirectory(string dirPath) - { - if (Directory.Exists(dirPath)) - { - string[] files = Directory.GetFiles(dirPath, "*.jpg"); - files = files.Concat(Directory.GetFiles(dirPath, "*.jpeg")).ToArray(); - files = files.Concat(Directory.GetFiles(dirPath, "*.png")).ToArray(); - files = files.Concat(Directory.GetFiles(dirPath, "*.tiff")).ToArray(); - files = files.Concat(Directory.GetFiles(dirPath, "*.tif")).ToArray(); - return files; - } - return new string[0]; - } - - /// - /// Raises the destroy event. - /// - void OnDestroy() - { - if (isImagesInputMode) - { - DisposeCalibraton(); - } - else - { - webCamTextureToMatHelper.Dispose(); - } - - Screen.orientation = ScreenOrientation.AutoRotation; - } - - /// - /// Raises the back button click event. - /// - public void OnBackButtonClick() - { - SceneManager.LoadScene("HoloLensWithOpenCVForUnityExample"); - } - - /// - /// Raises the play button click event. - /// - public void OnPlayButtonClick() - { - if (isImagesInputMode) return; - - webCamTextureToMatHelper.Play(); - } - - /// - /// Raises the pause button click event. - /// - public void OnPauseButtonClick() - { - if (isImagesInputMode) return; - - webCamTextureToMatHelper.Pause(); - } - - /// - /// Raises the stop button click event. - /// - public void OnStopButtonClick() - { - if (isImagesInputMode) return; - - webCamTextureToMatHelper.Stop(); - } - - /// - /// Raises the change camera button click event. - /// - public void OnChangeCameraButtonClick() - { - if (isImagesInputMode) return; - - webCamTextureToMatHelper.requestedIsFrontFacing = !webCamTextureToMatHelper.IsFrontFacing(); - } - - /// - /// Raises the marker type dropdown value changed event. - /// - public void OnMarkerTypeDropdownValueChanged(int result) - { - if ((int)markerType != result) - { - markerType = (MarkerType)result; - - dictionaryIdDropdown.interactable = (markerType == MarkerType.ChArUcoBoard); - - if (isImagesInputMode) - { - InitializeImagesInputMode(); - } - else - { - if (webCamTextureToMatHelper.IsInitialized()) - webCamTextureToMatHelper.Initialize(); - } - } - } - - /// - /// Raises the dictionary id dropdown value changed event. - /// - public void OnDictionaryIdDropdownValueChanged(int result) - { - if ((int)dictionaryId != result) - { - dictionaryId = (ArUcoDictionary)result; - dictionary = Aruco.getPredefinedDictionary((int)dictionaryId); - - if (isImagesInputMode) - { - InitializeImagesInputMode(); - } - else - { - if (webCamTextureToMatHelper.IsInitialized()) - webCamTextureToMatHelper.Initialize(); - } - } - } - - /// - /// Raises the squares X dropdown value changed event. - /// - public void OnSquaresXDropdownValueChanged(int result) - { - if ((int)squaresX != result + 1) - { - squaresX = (NumberOfSquaresX)(result + 1); - - if (isImagesInputMode) - { - InitializeImagesInputMode(); - } - else - { - if (webCamTextureToMatHelper.IsInitialized()) - webCamTextureToMatHelper.Initialize(); - } - } - } - - /// - /// Raises the squares Y dropdown value changed event. - /// - public void OnSquaresYDropdownValueChanged(int result) - { - if ((int)squaresY != result + 1) - { - squaresY = (NumberOfSquaresY)(result + 1); - - if (isImagesInputMode) - { - InitializeImagesInputMode(); - } - else - { - if (webCamTextureToMatHelper.IsInitialized()) - webCamTextureToMatHelper.Initialize(); - } - } - } - - /// - /// Raises the capture button click event. - /// - public void OnCaptureButtonClick() - { - if (isImagesInputMode) - { - if (!isCalibrating) - InitializeImagesInputMode(); - StartCoroutine("CalibrateCameraUsingImages"); - } - else - { - shouldCaptureFrame = true; - } - } - - /// - /// Raises the reset button click event. - /// - public void OnResetButtonClick() - { - if (isImagesInputMode) - { - if (!isCalibrating) - InitializeImagesInputMode(); - } - else - { - ResetCalibration(); - } - } - - /// - /// Raises the save button click event. - /// - public void OnSaveButtonClick() - { - string saveDirectoryPath = Path.Combine(Application.persistentDataPath, "HoloLensArUcoCameraCalibrationExample"); - - if (!Directory.Exists(saveDirectoryPath)) - { - Directory.CreateDirectory(saveDirectoryPath); - } - - string calibratonDirectoryName = "camera_parameters" + bgrMat.width() + "x" + bgrMat.height(); - string saveCalibratonFileDirectoryPath = Path.Combine(saveDirectoryPath, calibratonDirectoryName); - - // Clean up old files. - if (Directory.Exists(saveCalibratonFileDirectoryPath)) - { - DirectoryInfo directoryInfo = new DirectoryInfo(saveCalibratonFileDirectoryPath); - foreach (FileInfo fileInfo in directoryInfo.GetFiles()) - { - if ((fileInfo.Attributes & FileAttributes.ReadOnly) == FileAttributes.ReadOnly) - { - fileInfo.Attributes = FileAttributes.Normal; - } - } - if ((directoryInfo.Attributes & FileAttributes.ReadOnly) == FileAttributes.ReadOnly) - { - directoryInfo.Attributes = FileAttributes.Directory; - } - directoryInfo.Delete(true); - } - Directory.CreateDirectory(saveCalibratonFileDirectoryPath); - - // save the calibraton file. - string savePath = Path.Combine(saveCalibratonFileDirectoryPath, calibratonDirectoryName + ".xml"); - int frameCount = (markerType == MarkerType.ChArUcoBoard) ? allCorners.Count : imagePoints.Count; - CameraParameters param = new CameraParameters(frameCount, bgrMat.width(), bgrMat.height(), calibrationFlags, camMatrix, distCoeffs, repErr); - XmlSerializer serializer = new XmlSerializer(typeof(CameraParameters)); - using (var stream = new FileStream(savePath, FileMode.Create)) - { - serializer.Serialize(stream, param); - } - - // save the calibration images. -#if UNITY_WEBGL && !UNITY_EDITOR - string format = "jpg"; - MatOfInt compressionParams = new MatOfInt(Imgcodecs.IMWRITE_JPEG_QUALITY, 100); -#else - string format = "png"; - MatOfInt compressionParams = new MatOfInt(Imgcodecs.IMWRITE_PNG_COMPRESSION, 0); -#endif - for (int i = 0; i < allImgs.Count; ++i) - { - Imgcodecs.imwrite(Path.Combine(saveCalibratonFileDirectoryPath, calibratonDirectoryName + "_" + i.ToString("00") + "." + format), allImgs[i], compressionParams); - } - - savePathInputField.text = savePath; - Debug.Log("Saved the CameraParameters to disk in XML file format."); - Debug.Log("savePath: " + savePath); - } - - /// - /// Raises the tapped event. - /// - public void OnTapped(MixedRealityPointerEventData eventData) - { - Debug.Log("OnTapped!"); - - // Determine if a Gaze pointer is over a GUI. - if (eventData.selectedObject != null && (eventData.selectedObject.GetComponent