From 7b79b957c373e46ea73f560b5569fa91ece7dba7 Mon Sep 17 00:00:00 2001 From: SzabolcsGergely Date: Fri, 31 Mar 2023 16:21:48 +0300 Subject: [PATCH 01/87] Update FW: support for configurable ImageManip interpolation type --- depthai-core | 2 +- src/pipeline/datatype/ImageManipConfigBindings.cpp | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/depthai-core b/depthai-core index f605e5888..b3a1c3bee 160000 --- a/depthai-core +++ b/depthai-core @@ -1 +1 @@ -Subproject commit f605e5888754114e37545e2a517559c432d551ec +Subproject commit b3a1c3bee3a51235120bfb592d2c0aceadbefdad diff --git a/src/pipeline/datatype/ImageManipConfigBindings.cpp b/src/pipeline/datatype/ImageManipConfigBindings.cpp index e94ced202..885f1363c 100644 --- a/src/pipeline/datatype/ImageManipConfigBindings.cpp +++ b/src/pipeline/datatype/ImageManipConfigBindings.cpp @@ -21,6 +21,7 @@ void bind_imagemanipconfig(pybind11::module& m, void* pCallstack){ py::class_ rawImageManipCropConfig(rawImageManipConfig, "CropConfig", DOC(dai, RawImageManipConfig, CropConfig)); py::class_rawImageManipConfigResizeConfig(rawImageManipConfig, "ResizeConfig", DOC(dai, RawImageManipConfig, ResizeConfig)); py::class_ rawImageManipConfigFormatConfig(rawImageManipConfig, "FormatConfig", DOC(dai, RawImageManipConfig, FormatConfig)); + py::enum_ rawImageManipConfigInterpolation(rawImageManipConfig, "Interpolation", DOC(dai, RawImageManipConfig, Interpolation)); py::class_> imageManipConfig(m, "ImageManipConfig", DOC(dai, ImageManipConfig)); /////////////////////////////////////////////////////////////////////// @@ -45,6 +46,7 @@ void bind_imagemanipconfig(pybind11::module& m, void* pCallstack){ .def_readwrite("cropConfig", &RawImageManipConfig::cropConfig) .def_readwrite("resizeConfig", &RawImageManipConfig::resizeConfig) .def_readwrite("formatConfig", &RawImageManipConfig::formatConfig) + .def_readwrite("interpolation", &RawImageManipConfig::interpolation) ; rawImageManipConfigCropRect @@ -92,6 +94,12 @@ void bind_imagemanipconfig(pybind11::module& m, void* pCallstack){ .def_readwrite("flipVertical", &RawImageManipConfig::FormatConfig::flipVertical) ; + rawImageManipConfigInterpolation + .value("BILINEAR", RawImageManipConfig::Interpolation::BILINEAR) + .value("BICUBIC", RawImageManipConfig::Interpolation::BICUBIC) + .value("BYPASS", RawImageManipConfig::Interpolation::BYPASS) + ; + // Message imageManipConfig @@ -120,6 +128,7 @@ void bind_imagemanipconfig(pybind11::module& m, void* pCallstack){ .def("setReusePreviousImage", &ImageManipConfig::setReusePreviousImage, py::arg("reuse"), DOC(dai, ImageManipConfig, setReusePreviousImage)) .def("setSkipCurrentImage", &ImageManipConfig::setSkipCurrentImage, py::arg("skip"), DOC(dai, ImageManipConfig, setSkipCurrentImage)) .def("setKeepAspectRatio", &ImageManipConfig::setKeepAspectRatio, py::arg("keep"), DOC(dai, ImageManipConfig, setKeepAspectRatio)) + .def("setInterpolation", &ImageManipConfig::setInterpolation, py::arg("interpolation"), DOC(dai, ImageManipConfig, setInterpolation)) // getters .def("getCropXMin", &ImageManipConfig::getCropXMin, DOC(dai, ImageManipConfig, getCropXMin)) @@ -133,8 +142,10 @@ void bind_imagemanipconfig(pybind11::module& m, void* pCallstack){ .def("getFormatConfig", &ImageManipConfig::getFormatConfig, DOC(dai, ImageManipConfig, getFormatConfig)) .def("isResizeThumbnail", &ImageManipConfig::isResizeThumbnail, DOC(dai, ImageManipConfig, isResizeThumbnail)) .def("getColormap", &ImageManipConfig::getColormap, DOC(dai, ImageManipConfig, getColormap)) + .def("getInterpolation", &ImageManipConfig::getInterpolation, DOC(dai, ImageManipConfig, getInterpolation)) ; + m.attr("ImageManipConfig").attr("Interpolation") = rawImageManipConfigInterpolation; From fc62d363784ef6609bb2d017aa522aeb8432d82a Mon Sep 17 00:00:00 2001 From: SzabolcsGergely Date: Fri, 31 Mar 2023 19:19:43 +0300 Subject: [PATCH 02/87] Update core, bindings, API to use dai.Interpolation --- depthai-core | 2 +- docs/source/components/nodes/warp.rst | 2 +- examples/SpatialDetection/spatial_tiny_yolo.py | 4 ++-- examples/Warp/warp_mesh.py | 6 +++--- examples/Warp/warp_mesh_interactive.py | 2 +- src/pipeline/CommonBindings.cpp | 14 ++++++++++++++ src/pipeline/datatype/ImageManipConfigBindings.cpp | 10 ---------- src/pipeline/node/WarpBindings.cpp | 6 ------ 8 files changed, 22 insertions(+), 24 deletions(-) diff --git a/depthai-core b/depthai-core index b3a1c3bee..9be2ff9d7 160000 --- a/depthai-core +++ b/depthai-core @@ -1 +1 @@ -Subproject commit b3a1c3bee3a51235120bfb592d2c0aceadbefdad +Subproject commit 9be2ff9d7048758291f2fd45e39898e04b2d84c1 diff --git a/docs/source/components/nodes/warp.rst b/docs/source/components/nodes/warp.rst index b707990d0..8a929aea7 100644 --- a/docs/source/components/nodes/warp.rst +++ b/docs/source/components/nodes/warp.rst @@ -65,7 +65,7 @@ Usage # Warp engines to be used (0,1,2) warp.setHwIds([1]) # Warp interpolation mode, choose between BILINEAR, BICUBIC, BYPASS - warp.setInterpolation(dai.node.Warp.Properties.Interpolation.BYPASS) + warp.setInterpolation(dai.Interpolation.NEAREST_NEIGHBOR) .. code-tab:: c++ diff --git a/examples/SpatialDetection/spatial_tiny_yolo.py b/examples/SpatialDetection/spatial_tiny_yolo.py index 5575bccd6..c4d2d0575 100755 --- a/examples/SpatialDetection/spatial_tiny_yolo.py +++ b/examples/SpatialDetection/spatial_tiny_yolo.py @@ -122,7 +122,7 @@ previewQueue = device.getOutputQueue(name="rgb", maxSize=4, blocking=False) detectionNNQueue = device.getOutputQueue(name="detections", maxSize=4, blocking=False) depthQueue = device.getOutputQueue(name="depth", maxSize=4, blocking=False) - networkQueue = device.getOutputQueue(name="nnNetwork", maxSize=4, blocking=False); + networkQueue = device.getOutputQueue(name="nnNetwork", maxSize=4, blocking=False) startTime = time.monotonic() counter = 0 @@ -141,7 +141,7 @@ for ten in inNN.getAllLayerNames(): toPrint = f'{toPrint} {ten},' print(toPrint) - printOutputLayersOnce = False; + printOutputLayersOnce = False frame = inPreview.getCvFrame() depthFrame = depth.getFrame() # depthFrame values are in millimeters diff --git a/examples/Warp/warp_mesh.py b/examples/Warp/warp_mesh.py index c0bdc0c3e..f608dfafe 100755 --- a/examples/Warp/warp_mesh.py +++ b/examples/Warp/warp_mesh.py @@ -25,7 +25,7 @@ warp1.setOutputSize(WARP1_OUTPUT_FRAME_SIZE) warp1.setMaxOutputFrameSize(WARP1_OUTPUT_FRAME_SIZE[0] * WARP1_OUTPUT_FRAME_SIZE[1] * 3) warp1.setHwIds([1]) -warp1.setInterpolation(dai.node.Warp.Properties.Interpolation.BYPASS) +warp1.setInterpolation(dai.Interpolation.NEAREST_NEIGHBOR) camRgb.preview.link(warp1.inputImage) xout1 = pipeline.create(dai.node.XLinkOut) @@ -43,9 +43,9 @@ warp2.setWarpMesh(mesh2, 3, 3) warp2.setMaxOutputFrameSize(maxFrameSize) warp1.setHwIds([2]) -warp2.setInterpolation(dai.node.Warp.Properties.Interpolation.BICUBIC) +warp2.setInterpolation(dai.Interpolation.BICUBIC) -camRgb.preview.link(warp2.inputImage); +camRgb.preview.link(warp2.inputImage) xout2 = pipeline.create(dai.node.XLinkOut) xout2.setStreamName('out2') warp2.out.link(xout2.input) diff --git a/examples/Warp/warp_mesh_interactive.py b/examples/Warp/warp_mesh_interactive.py index 6065dd951..befe5832b 100755 --- a/examples/Warp/warp_mesh_interactive.py +++ b/examples/Warp/warp_mesh_interactive.py @@ -74,7 +74,7 @@ def create_pipeline(mesh): camRgb.preview.link(warp.inputImage) warp.setHwIds([1]) - warp.setInterpolation(dai.node.Warp.Properties.Interpolation.BYPASS) + warp.setInterpolation(dai.Interpolation.NEAREST_NEIGHBOR) # Output warped xout_warped = pipeline.create(dai.node.XLinkOut) xout_warped.setStreamName('warped') diff --git a/src/pipeline/CommonBindings.cpp b/src/pipeline/CommonBindings.cpp index abb40023c..08663e1b5 100644 --- a/src/pipeline/CommonBindings.cpp +++ b/src/pipeline/CommonBindings.cpp @@ -21,6 +21,7 @@ #include "depthai-shared/common/Rect.hpp" #include "depthai-shared/common/Colormap.hpp" #include "depthai-shared/common/FrameEvent.hpp" +#include "depthai-shared/common/Interpolation.hpp" // depthai #include "depthai/common/CameraFeatures.hpp" @@ -57,6 +58,7 @@ void CommonBindings::bind(pybind11::module& m, void* pCallstack){ py::enum_ cameraExposureOffset(m, "CameraExposureOffset"); py::enum_ colormap(m, "Colormap", DOC(dai, Colormap)); py::enum_ frameEvent(m, "FrameEvent", DOC(dai, FrameEvent)); + py::enum_ interpolation(m, "Interpolation", DOC(dai, Interpolation)); /////////////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////////// @@ -347,5 +349,17 @@ void CommonBindings::bind(pybind11::module& m, void* pCallstack){ .value("READOUT_START", FrameEvent::READOUT_START) .value("READOUT_END", FrameEvent::READOUT_END) ; + interpolation + .value("BILINEAR", Interpolation::BILINEAR) + .value("BICUBIC", Interpolation::BICUBIC) + .value("NEAREST_NEIGHBOR", Interpolation::NEAREST_NEIGHBOR) + .value("BYPASS", Interpolation::BYPASS) + ; + + m.attr("DEFAULT_INTERPOLATION") = dai::DEFAULT_INTERPOLATION; + m.attr("DEFAULT_DISPARITY_DEPTH_INTERPOLATION") = dai::DEFAULT_DISPARITY_DEPTH_INTERPOLATION; + //backward compatibility + m.attr("node").attr("Warp").attr("Properties").attr("Interpolation") = interpolation; + } diff --git a/src/pipeline/datatype/ImageManipConfigBindings.cpp b/src/pipeline/datatype/ImageManipConfigBindings.cpp index 885f1363c..300cd9fa2 100644 --- a/src/pipeline/datatype/ImageManipConfigBindings.cpp +++ b/src/pipeline/datatype/ImageManipConfigBindings.cpp @@ -21,7 +21,6 @@ void bind_imagemanipconfig(pybind11::module& m, void* pCallstack){ py::class_ rawImageManipCropConfig(rawImageManipConfig, "CropConfig", DOC(dai, RawImageManipConfig, CropConfig)); py::class_rawImageManipConfigResizeConfig(rawImageManipConfig, "ResizeConfig", DOC(dai, RawImageManipConfig, ResizeConfig)); py::class_ rawImageManipConfigFormatConfig(rawImageManipConfig, "FormatConfig", DOC(dai, RawImageManipConfig, FormatConfig)); - py::enum_ rawImageManipConfigInterpolation(rawImageManipConfig, "Interpolation", DOC(dai, RawImageManipConfig, Interpolation)); py::class_> imageManipConfig(m, "ImageManipConfig", DOC(dai, ImageManipConfig)); /////////////////////////////////////////////////////////////////////// @@ -94,12 +93,6 @@ void bind_imagemanipconfig(pybind11::module& m, void* pCallstack){ .def_readwrite("flipVertical", &RawImageManipConfig::FormatConfig::flipVertical) ; - rawImageManipConfigInterpolation - .value("BILINEAR", RawImageManipConfig::Interpolation::BILINEAR) - .value("BICUBIC", RawImageManipConfig::Interpolation::BICUBIC) - .value("BYPASS", RawImageManipConfig::Interpolation::BYPASS) - ; - // Message imageManipConfig @@ -145,8 +138,5 @@ void bind_imagemanipconfig(pybind11::module& m, void* pCallstack){ .def("getInterpolation", &ImageManipConfig::getInterpolation, DOC(dai, ImageManipConfig, getInterpolation)) ; - m.attr("ImageManipConfig").attr("Interpolation") = rawImageManipConfigInterpolation; - - } diff --git a/src/pipeline/node/WarpBindings.cpp b/src/pipeline/node/WarpBindings.cpp index 954321f19..4e090834e 100644 --- a/src/pipeline/node/WarpBindings.cpp +++ b/src/pipeline/node/WarpBindings.cpp @@ -13,7 +13,6 @@ void bind_warp(pybind11::module& m, void* pCallstack){ // Node and Properties declare upfront // TODO(themarpe) - properties py::class_ warpProperties(m, "WarpProperties", DOC(dai, WarpProperties)); - py::enum_ warpPropertiesInterpolation(warpProperties, "Interpolation", DOC(dai, WarpProperties, Interpolation)); auto warp = ADD_NODE(Warp); /////////////////////////////////////////////////////////////////////// @@ -29,11 +28,6 @@ void bind_warp(pybind11::module& m, void* pCallstack){ /////////////////////////////////////////////////////////////////////// // properties - warpPropertiesInterpolation - .value("BILINEAR", Warp::Properties::Interpolation::BILINEAR) - .value("BICUBIC", Warp::Properties::Interpolation::BICUBIC) - .value("BYPASS", Warp::Properties::Interpolation::BYPASS) - ; // ImageManip Node warp From 4e3af17ecd997c3bb0effd0eb8935a71f10e42ac Mon Sep 17 00:00:00 2001 From: SzabolcsGergely Date: Fri, 31 Mar 2023 21:30:57 +0300 Subject: [PATCH 03/87] Update FW with latest develop fixes --- depthai-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/depthai-core b/depthai-core index 8cea0941f..b2c65a88e 160000 --- a/depthai-core +++ b/depthai-core @@ -1 +1 @@ -Subproject commit 8cea0941f5c8692ae4877493bf77e5fc53819609 +Subproject commit b2c65a88ebe862bcbccfe30f8fc90c8183750549 From 913a4f3796a876c753897efe6726f3c8c1bc51d4 Mon Sep 17 00:00:00 2001 From: SzabolcsGergely Date: Fri, 31 Mar 2023 23:26:21 +0300 Subject: [PATCH 04/87] Refactor default interpolation naming --- depthai-core | 2 +- src/pipeline/CommonBindings.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/depthai-core b/depthai-core index b2c65a88e..15bb02f41 160000 --- a/depthai-core +++ b/depthai-core @@ -1 +1 @@ -Subproject commit b2c65a88ebe862bcbccfe30f8fc90c8183750549 +Subproject commit 15bb02f4165a25d7201d6182f34cbf0ff0ed9eee diff --git a/src/pipeline/CommonBindings.cpp b/src/pipeline/CommonBindings.cpp index 08663e1b5..71132ae27 100644 --- a/src/pipeline/CommonBindings.cpp +++ b/src/pipeline/CommonBindings.cpp @@ -354,10 +354,10 @@ void CommonBindings::bind(pybind11::module& m, void* pCallstack){ .value("BICUBIC", Interpolation::BICUBIC) .value("NEAREST_NEIGHBOR", Interpolation::NEAREST_NEIGHBOR) .value("BYPASS", Interpolation::BYPASS) + .value("DEFAULT", Interpolation::DEFAULT) + .value("DEFAULT_DISPARITY_DEPTH", Interpolation::DEFAULT_DISPARITY_DEPTH) ; - m.attr("DEFAULT_INTERPOLATION") = dai::DEFAULT_INTERPOLATION; - m.attr("DEFAULT_DISPARITY_DEPTH_INTERPOLATION") = dai::DEFAULT_DISPARITY_DEPTH_INTERPOLATION; //backward compatibility m.attr("node").attr("Warp").attr("Properties").attr("Interpolation") = interpolation; From 7942c5a72a9e6901036dae0b8f65462a1fb79b9e Mon Sep 17 00:00:00 2001 From: Andrej Susnik Date: Wed, 7 Jun 2023 10:06:41 +0200 Subject: [PATCH 05/87] Add example for readCalibration2 in script node --- examples/Script/script_read_calibration.py | 27 ++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 examples/Script/script_read_calibration.py diff --git a/examples/Script/script_read_calibration.py b/examples/Script/script_read_calibration.py new file mode 100644 index 000000000..032de24b7 --- /dev/null +++ b/examples/Script/script_read_calibration.py @@ -0,0 +1,27 @@ +import cv2 +import depthai as dai + +# Start defining a pipeline +pipeline = dai.Pipeline() + +# Script node +script = pipeline.create(dai.node.Script) +script.setProcessor(dai.ProcessorType.LEON_CSS) +script.setScript(""" + + cal = Device.readCalibration2() + left_camera_id = cal.getStereoLeftCameraId() + right_camera_id = cal.getStereoRightCameraId() + + extrinsics = cal.getCameraExtrinsics(left_camera_id, right_camera_id) + intrinsics_left = cal.getCameraIntrinsics(left_camera_id) + + print(extrinsics) + print(intrinsics_left) + +""") + +# Connect to device with pipeline +with dai.Device(pipeline) as device: + while True: + pass \ No newline at end of file From 3cffd3bf700adca5789c88d0d37fa58a881b5575 Mon Sep 17 00:00:00 2001 From: Andrej Susnik Date: Wed, 7 Jun 2023 20:09:27 +0200 Subject: [PATCH 06/87] Wait for pipeline to finish in read calibration example --- examples/Script/script_read_calibration.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/examples/Script/script_read_calibration.py b/examples/Script/script_read_calibration.py index 032de24b7..d247018ee 100644 --- a/examples/Script/script_read_calibration.py +++ b/examples/Script/script_read_calibration.py @@ -1,4 +1,4 @@ -import cv2 +#!/usr/bin/env python3 import depthai as dai # Start defining a pipeline @@ -8,6 +8,7 @@ script = pipeline.create(dai.node.Script) script.setProcessor(dai.ProcessorType.LEON_CSS) script.setScript(""" + import time cal = Device.readCalibration2() left_camera_id = cal.getStereoLeftCameraId() @@ -19,9 +20,14 @@ print(extrinsics) print(intrinsics_left) + time.sleep(1) + node.io['end'].send(Buffer(32)) """) +xout = pipeline.create(dai.node.XLinkOut) +xout.setStreamName('end') +script.outputs['end'].link(xout.input) + # Connect to device with pipeline with dai.Device(pipeline) as device: - while True: - pass \ No newline at end of file + device.getOutputQueue('end').get() \ No newline at end of file From 1a430f09e621d3b2dc11d1aef73cbeb66558aca3 Mon Sep 17 00:00:00 2001 From: TheMarpe Date: Wed, 14 Jun 2023 19:14:05 +0200 Subject: [PATCH 07/87] Fixed dispatch to hil tests --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 469f0b65d..ea091c49e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -560,12 +560,12 @@ jobs: uses: codex-/return-dispatch@v1 id: return_dispatch with: - token: ${{ secrets.HIL_CORE_DISPATCH_TOKEN }} # Note this is NOT GITHUB_TOKEN but a PAT + token: ${{ secrets.HIL_CORE_DISPATCH_TOKEN }} # Note this is NOT GITHUB_TOKEN but a PAT ref: main # or refs/heads/target_branch repo: depthai-core-hil-tests owner: luxonis workflow: regression_test.yml - workflow_inputs: '{"commit": "${{ github.ref }}", "sha": "${{ github.sha }}", "parent_url": "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"}' + workflow_inputs: '{"commit": "${{ github.ref }}", "parent_url": "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"}' workflow_timeout_seconds: 120 # Default: 300 - name: Release From c9e3e2403e9f625e642031fcbcc369579652913d Mon Sep 17 00:00:00 2001 From: Andrej Susnik Date: Tue, 20 Jun 2023 12:39:14 +0200 Subject: [PATCH 08/87] Replace print with node.info in script node read calibartion example --- examples/Script/script_read_calibration.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/Script/script_read_calibration.py b/examples/Script/script_read_calibration.py index d247018ee..1501a12d5 100644 --- a/examples/Script/script_read_calibration.py +++ b/examples/Script/script_read_calibration.py @@ -17,8 +17,8 @@ extrinsics = cal.getCameraExtrinsics(left_camera_id, right_camera_id) intrinsics_left = cal.getCameraIntrinsics(left_camera_id) - print(extrinsics) - print(intrinsics_left) + node.info(extrinsics.__str__()) + node.info(intrinsics_left.__str__()) time.sleep(1) node.io['end'].send(Buffer(32)) From 02ae1adfee67c589d7d561dd95c60d7e0f858fed Mon Sep 17 00:00:00 2001 From: Martin Peterlin Date: Wed, 21 Jun 2023 02:49:58 +0200 Subject: [PATCH 09/87] Tweak existing armv7l wheels to be presented with armv7l & armv6l tags (built on RPi, toolchains target armv6l by default) --- .github/workflows/main.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ea091c49e..562691e2e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -156,11 +156,18 @@ jobs: run: python3 -m pip wheel . -w ./wheelhouse/ --verbose - name: Auditing wheel run: for whl in wheelhouse/*.whl; do auditwheel repair "$whl" --plat linux_armv7l -w wheelhouse/audited/; done + - name: Install tweaked auditwheel and add armv6l tag + run: | + python3 -m venv newauditvenv + source newauditvenv/bin/activate + python3 -m pip install git+https://github.com/luxonis/auditwheel@main + for whl in wheelhouse/*.whl; do python3 -m auditwheel addtag -t linux_armv7l linux_armv6l -w wheelhouse/postaudited/ "$whl"; done + deactivate - name: Archive wheel artifacts uses: actions/upload-artifact@v3 with: name: audited-wheels - path: wheelhouse/audited/ + path: wheelhouse/postaudited/ - name: Deploy wheels to artifactory (if not a release) if: startsWith(github.ref, 'refs/tags/v') != true run: bash ./ci/upload-artifactory.sh From 9d9c07a956a2312a78264ad10b251732a9d8551c Mon Sep 17 00:00:00 2001 From: SzabolcsGergely Date: Wed, 21 Jun 2023 21:12:00 +0300 Subject: [PATCH 10/87] ToF: add median filter support --- depthai-core | 2 +- examples/StereoDepth/stereo_depth_from_host.py | 3 +++ src/pipeline/datatype/ToFConfigBindings.cpp | 3 ++- utilities/cam_test.py | 12 ++++++++++++ 4 files changed, 18 insertions(+), 2 deletions(-) diff --git a/depthai-core b/depthai-core index 44586d641..b0e37dcd6 160000 --- a/depthai-core +++ b/depthai-core @@ -1 +1 @@ -Subproject commit 44586d641e53c1c122699c4fd46245bbb49fec70 +Subproject commit b0e37dcd6256d54560f91755bb06c8a6bacabf81 diff --git a/examples/StereoDepth/stereo_depth_from_host.py b/examples/StereoDepth/stereo_depth_from_host.py index bf5328785..7662e35f6 100755 --- a/examples/StereoDepth/stereo_depth_from_host.py +++ b/examples/StereoDepth/stereo_depth_from_host.py @@ -588,6 +588,9 @@ def __init__(self, config): fov = 71.86 focal = width / (2 * math.tan(fov / 2 / 180 * math.pi)) +stereo.setBaseline(baseline/10) +stereo.setFocalLength(focal) + streams = ['left', 'right'] if outRectified: streams.extend(['rectified_left', 'rectified_right']) diff --git a/src/pipeline/datatype/ToFConfigBindings.cpp b/src/pipeline/datatype/ToFConfigBindings.cpp index a51fae8ac..3e2aaf3c7 100644 --- a/src/pipeline/datatype/ToFConfigBindings.cpp +++ b/src/pipeline/datatype/ToFConfigBindings.cpp @@ -48,10 +48,10 @@ void bind_tofconfig(pybind11::module& m, void* pCallstack){ depthParams .def(py::init<>()) - .def_readwrite("enable", &RawToFConfig::DepthParams::enable, DOC(dai, RawToFConfig, DepthParams, enable)) .def_readwrite("freqModUsed", &RawToFConfig::DepthParams::freqModUsed, DOC(dai, RawToFConfig, DepthParams, freqModUsed)) .def_readwrite("avgPhaseShuffle", &RawToFConfig::DepthParams::avgPhaseShuffle, DOC(dai, RawToFConfig, DepthParams, avgPhaseShuffle)) .def_readwrite("minimumAmplitude", &RawToFConfig::DepthParams::minimumAmplitude, DOC(dai, RawToFConfig, DepthParams, minimumAmplitude)) + .def_readwrite("median", &RawToFConfig::DepthParams::median, DOC(dai, RawToFConfig, DepthParams, median)) ; // Message @@ -63,6 +63,7 @@ void bind_tofconfig(pybind11::module& m, void* pCallstack){ .def("setFreqModUsed", static_cast(&ToFConfig::setFreqModUsed), DOC(dai, ToFConfig, setFreqModUsed)) .def("setAvgPhaseShuffle", &ToFConfig::setAvgPhaseShuffle, DOC(dai, ToFConfig, setAvgPhaseShuffle)) .def("setMinAmplitude", &ToFConfig::setMinAmplitude, DOC(dai, ToFConfig, setMinAmplitude)) + .def("setMedianFilter", &ToFConfig::setMedianFilter, DOC(dai, ToFConfig, setMedianFilter)) .def("set", &ToFConfig::set, py::arg("config"), DOC(dai, ToFConfig, set)) .def("get", &ToFConfig::get, DOC(dai, ToFConfig, get)) diff --git a/utilities/cam_test.py b/utilities/cam_test.py index 240007de3..b3e478830 100755 --- a/utilities/cam_test.py +++ b/utilities/cam_test.py @@ -89,6 +89,8 @@ def socket_type_pair(arg): help="Show also ToF amplitude output alongside depth") parser.add_argument('-tofcm', '--tof-cm', action='store_true', help="Show ToF depth output in centimeters, capped to 255") +parser.add_argument('-tofmedian', '--tof-median', choices=[0,3,5,7], default=5, type=int, + help="ToF median filter kernel size") parser.add_argument('-rgbprev', '--rgb-preview', action='store_true', help="Show RGB `preview` stream instead of full size `isp`") @@ -221,6 +223,16 @@ def get(self): tofConfig.depthParams.freqModUsed = dai.RawToFConfig.DepthParams.TypeFMod.MIN tofConfig.depthParams.avgPhaseShuffle = False tofConfig.depthParams.minimumAmplitude = 3.0 + + if args.tof_median == 0: + tofConfig.depthParams.median = dai.MedianFilter.MEDIAN_OFF + elif args.tof_median == 3: + tofConfig.depthParams.median = dai.MedianFilter.KERNEL_3x3 + elif args.tof_median == 5: + tofConfig.depthParams.median = dai.MedianFilter.KERNEL_5x5 + elif args.tof_median == 7: + tofConfig.depthParams.median = dai.MedianFilter.KERNEL_7x7 + tof[c].initialConfig.set(tofConfig) if args.tof_amplitude: amp_name = 'tof_amplitude_' + c From 9d15028afa97452ce82c9cde78d658bf48125b21 Mon Sep 17 00:00:00 2001 From: TheMarpe Date: Thu, 22 Jun 2023 21:19:08 +0200 Subject: [PATCH 11/87] Pinned mypy dependency for Python wheel build --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 6e22a4b62..0a0c03a15 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,2 +1,2 @@ [build-system] -requires = ["setuptools", "wheel", "mypy", "cmake==3.25"] \ No newline at end of file +requires = ["setuptools", "wheel", "mypy==1.3.0", "cmake==3.25"] \ No newline at end of file From 8b076a99b4e80d007b60a5ab557ddc24d87aaf3b Mon Sep 17 00:00:00 2001 From: TheMarpe Date: Fri, 23 Jun 2023 01:00:01 +0200 Subject: [PATCH 12/87] Simplified armv6l CI --- .github/workflows/main.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 562691e2e..6afb37244 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -158,11 +158,8 @@ jobs: run: for whl in wheelhouse/*.whl; do auditwheel repair "$whl" --plat linux_armv7l -w wheelhouse/audited/; done - name: Install tweaked auditwheel and add armv6l tag run: | - python3 -m venv newauditvenv - source newauditvenv/bin/activate python3 -m pip install git+https://github.com/luxonis/auditwheel@main for whl in wheelhouse/*.whl; do python3 -m auditwheel addtag -t linux_armv7l linux_armv6l -w wheelhouse/postaudited/ "$whl"; done - deactivate - name: Archive wheel artifacts uses: actions/upload-artifact@v3 with: From 972c00eea4b4f6818a00273c36d5445beaa1b0ca Mon Sep 17 00:00:00 2001 From: JanLipovsek Date: Mon, 26 Jun 2023 17:58:07 +0200 Subject: [PATCH 13/87] updated depth examples to not fail on empty depth detections --- examples/SpatialDetection/spatial_calculator_multi_roi.py | 5 ++++- examples/SpatialDetection/spatial_location_calculator.py | 5 ++++- examples/SpatialDetection/spatial_mobilenet.py | 5 ++++- examples/SpatialDetection/spatial_mobilenet_mono.py | 5 ++++- examples/SpatialDetection/spatial_tiny_yolo.py | 5 ++++- examples/StereoDepth/depth_crop_control.py | 5 ++++- 6 files changed, 24 insertions(+), 6 deletions(-) diff --git a/examples/SpatialDetection/spatial_calculator_multi_roi.py b/examples/SpatialDetection/spatial_calculator_multi_roi.py index 24d29020f..c813f17cb 100755 --- a/examples/SpatialDetection/spatial_calculator_multi_roi.py +++ b/examples/SpatialDetection/spatial_calculator_multi_roi.py @@ -67,7 +67,10 @@ depthFrame = inDepth.getFrame() # depthFrame values are in millimeters depth_downscaled = depthFrame[::4] - min_depth = np.percentile(depth_downscaled[depth_downscaled != 0], 1) + if np.all(depth_downscaled == 0): + min_depth = 0 # Set a default minimum depth value when all elements are zero + else: + min_depth = np.percentile(depth_downscaled[depth_downscaled != 0], 1) max_depth = np.percentile(depth_downscaled, 99) depthFrameColor = np.interp(depthFrame, (min_depth, max_depth), (0, 255)).astype(np.uint8) depthFrameColor = cv2.applyColorMap(depthFrameColor, cv2.COLORMAP_HOT) diff --git a/examples/SpatialDetection/spatial_location_calculator.py b/examples/SpatialDetection/spatial_location_calculator.py index 2e35e409f..89260cd6f 100755 --- a/examples/SpatialDetection/spatial_location_calculator.py +++ b/examples/SpatialDetection/spatial_location_calculator.py @@ -75,7 +75,10 @@ depthFrame = inDepth.getFrame() # depthFrame values are in millimeters depth_downscaled = depthFrame[::4] - min_depth = np.percentile(depth_downscaled[depth_downscaled != 0], 1) + if np.all(depth_downscaled == 0): + min_depth = 0 # Set a default minimum depth value when all elements are zero + else: + min_depth = np.percentile(depth_downscaled[depth_downscaled != 0], 1) max_depth = np.percentile(depth_downscaled, 99) depthFrameColor = np.interp(depthFrame, (min_depth, max_depth), (0, 255)).astype(np.uint8) depthFrameColor = cv2.applyColorMap(depthFrameColor, cv2.COLORMAP_HOT) diff --git a/examples/SpatialDetection/spatial_mobilenet.py b/examples/SpatialDetection/spatial_mobilenet.py index 42643b207..2c19db7db 100755 --- a/examples/SpatialDetection/spatial_mobilenet.py +++ b/examples/SpatialDetection/spatial_mobilenet.py @@ -115,7 +115,10 @@ depthFrame = depth.getFrame() # depthFrame values are in millimeters depth_downscaled = depthFrame[::4] - min_depth = np.percentile(depth_downscaled[depth_downscaled != 0], 1) + if np.all(depth_downscaled == 0): + min_depth = 0 # Set a default minimum depth value when all elements are zero + else: + min_depth = np.percentile(depth_downscaled[depth_downscaled != 0], 1) max_depth = np.percentile(depth_downscaled, 99) depthFrameColor = np.interp(depthFrame, (min_depth, max_depth), (0, 255)).astype(np.uint8) depthFrameColor = cv2.applyColorMap(depthFrameColor, cv2.COLORMAP_HOT) diff --git a/examples/SpatialDetection/spatial_mobilenet_mono.py b/examples/SpatialDetection/spatial_mobilenet_mono.py index 8c7634a72..be9e80fe0 100755 --- a/examples/SpatialDetection/spatial_mobilenet_mono.py +++ b/examples/SpatialDetection/spatial_mobilenet_mono.py @@ -118,7 +118,10 @@ depthFrame = inDepth.getFrame() # depthFrame values are in millimeters depth_downscaled = depthFrame[::4] - min_depth = np.percentile(depth_downscaled[depth_downscaled != 0], 1) + if np.all(depth_downscaled == 0): + min_depth = 0 # Set a default minimum depth value when all elements are zero + else: + min_depth = np.percentile(depth_downscaled[depth_downscaled != 0], 1) max_depth = np.percentile(depth_downscaled, 99) depthFrameColor = np.interp(depthFrame, (min_depth, max_depth), (0, 255)).astype(np.uint8) depthFrameColor = cv2.applyColorMap(depthFrameColor, cv2.COLORMAP_HOT) diff --git a/examples/SpatialDetection/spatial_tiny_yolo.py b/examples/SpatialDetection/spatial_tiny_yolo.py index 6350ca9f1..88f1f606f 100755 --- a/examples/SpatialDetection/spatial_tiny_yolo.py +++ b/examples/SpatialDetection/spatial_tiny_yolo.py @@ -148,7 +148,10 @@ depthFrame = depth.getFrame() # depthFrame values are in millimeters depth_downscaled = depthFrame[::4] - min_depth = np.percentile(depth_downscaled[depth_downscaled != 0], 1) + if np.all(depth_downscaled == 0): + min_depth = 0 # Set a default minimum depth value when all elements are zero + else: + min_depth = np.percentile(depth_downscaled[depth_downscaled != 0], 1) max_depth = np.percentile(depth_downscaled, 99) depthFrameColor = np.interp(depthFrame, (min_depth, max_depth), (0, 255)).astype(np.uint8) depthFrameColor = cv2.applyColorMap(depthFrameColor, cv2.COLORMAP_HOT) diff --git a/examples/StereoDepth/depth_crop_control.py b/examples/StereoDepth/depth_crop_control.py index 70dcb5a11..9236142cd 100755 --- a/examples/StereoDepth/depth_crop_control.py +++ b/examples/StereoDepth/depth_crop_control.py @@ -64,7 +64,10 @@ # Frame is transformed, the color map will be applied to highlight the depth info depth_downscaled = depthFrame[::4] - min_depth = np.percentile(depth_downscaled[depth_downscaled != 0], 1) + if np.all(depth_downscaled == 0): + min_depth = 0 # Set a default minimum depth value when all elements are zero + else: + min_depth = np.percentile(depth_downscaled[depth_downscaled != 0], 1) max_depth = np.percentile(depth_downscaled, 99) depthFrameColor = np.interp(depthFrame, (min_depth, max_depth), (0, 255)).astype(np.uint8) depthFrameColor = cv2.applyColorMap(depthFrameColor, cv2.COLORMAP_HOT) From a35b488836a030d9d2f72952917f0084c323f0ee Mon Sep 17 00:00:00 2001 From: JanLipovsek Date: Mon, 26 Jun 2023 17:58:46 +0200 Subject: [PATCH 14/87] increased timeout to hiltest dispatch --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ea091c49e..c27361523 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -566,7 +566,7 @@ jobs: owner: luxonis workflow: regression_test.yml workflow_inputs: '{"commit": "${{ github.ref }}", "parent_url": "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"}' - workflow_timeout_seconds: 120 # Default: 300 + workflow_timeout_seconds: 300 # was 120 Default: 300 - name: Release run: echo "https://github.com/luxonis/depthai-core-hil-tests/actions/runs/${{steps.return_dispatch.outputs.run_id}}" >> $GITHUB_STEP_SUMMARY \ No newline at end of file From 1fe15f2191fdb9c90305c4136f5b118fa3b9f3c0 Mon Sep 17 00:00:00 2001 From: SzabolcsGergely Date: Mon, 26 Jun 2023 22:13:19 +0300 Subject: [PATCH 15/87] Update core to latest develop --- depthai-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/depthai-core b/depthai-core index 8b3daa5fb..733fbe0e4 160000 --- a/depthai-core +++ b/depthai-core @@ -1 +1 @@ -Subproject commit 8b3daa5fb1a76e07608c7dfbebd88b608ec9a430 +Subproject commit 733fbe0e48740372c6befa9e8c98ccce2523102f From 394422980f8156f514d53fadf13d977c4e5c4beb Mon Sep 17 00:00:00 2001 From: JanLipovsek Date: Tue, 4 Jul 2023 11:33:58 +0200 Subject: [PATCH 16/87] added check for empty depth frame --- examples/mixed/rotated_spatial_detections.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/examples/mixed/rotated_spatial_detections.py b/examples/mixed/rotated_spatial_detections.py index c22dd057b..a0e668acf 100755 --- a/examples/mixed/rotated_spatial_detections.py +++ b/examples/mixed/rotated_spatial_detections.py @@ -106,7 +106,10 @@ depthFrame = depth.getFrame() # depthFrame values are in millimeters depth_downscaled = depthFrame[::4] - min_depth = np.percentile(depth_downscaled[depth_downscaled != 0], 1) + if np.all(depth_downscaled == 0): + min_depth = 0 # Set a default minimum depth value when all elements are zero + else: + min_depth = np.percentile(depth_downscaled[depth_downscaled != 0], 1) max_depth = np.percentile(depth_downscaled, 99) depthFrameColor = np.interp(depthFrame, (min_depth, max_depth), (0, 255)).astype(np.uint8) depthFrameColor = cv2.applyColorMap(depthFrameColor, cv2.COLORMAP_HOT) From 8154b7477240b2db854a9715d6424d3724888057 Mon Sep 17 00:00:00 2001 From: SzabolcsGergely Date: Tue, 4 Jul 2023 16:11:23 +0300 Subject: [PATCH 17/87] FW: Stereo: Apply alpha scaling parameter to RGB intrinsics when RGB alignment is enabled --- depthai-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/depthai-core b/depthai-core index 733fbe0e4..e52f10f2d 160000 --- a/depthai-core +++ b/depthai-core @@ -1 +1 @@ -Subproject commit 733fbe0e48740372c6befa9e8c98ccce2523102f +Subproject commit e52f10f2d9933f4e4a543bc02d70683147080f3a From 6f15fe134453e09384a259d2a6bf23728e34734f Mon Sep 17 00:00:00 2001 From: alex-luxonis Date: Tue, 4 Jul 2023 15:53:57 +0300 Subject: [PATCH 18/87] FW: fix CAM_D enumeration on OAK-FFC-4P R7 --- depthai-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/depthai-core b/depthai-core index e52f10f2d..1702d3134 160000 --- a/depthai-core +++ b/depthai-core @@ -1 +1 @@ -Subproject commit e52f10f2d9933f4e4a543bc02d70683147080f3a +Subproject commit 1702d31345357cdd9a856badd3d27dd76cc27dfc From eb0056f4dc2e2f05f2c3cf997d206153e3e7b5d5 Mon Sep 17 00:00:00 2001 From: SzabolcsGergely Date: Tue, 4 Jul 2023 19:39:26 +0300 Subject: [PATCH 19/87] Camera: move alpha parameter as optional to be consistent with stereo --- depthai-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/depthai-core b/depthai-core index e52f10f2d..3ef117594 160000 --- a/depthai-core +++ b/depthai-core @@ -1 +1 @@ -Subproject commit e52f10f2d9933f4e4a543bc02d70683147080f3a +Subproject commit 3ef1175944637c928ac571c9b1f5cff92161bfde From 9326a00be72b922aa2a5dda4298849635a0a14fb Mon Sep 17 00:00:00 2001 From: TheMarpe Date: Tue, 4 Jul 2023 20:53:34 +0200 Subject: [PATCH 20/87] Modified mypy requirement --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0a0c03a15..d770236f7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,2 +1,2 @@ [build-system] -requires = ["setuptools", "wheel", "mypy==1.3.0", "cmake==3.25"] \ No newline at end of file +requires = ["setuptools", "wheel", "mypy<=1.3.0", "cmake==3.25"] \ No newline at end of file From e5573f0d5a7aa4b3ae5ead5f1f21213f72ab6ff3 Mon Sep 17 00:00:00 2001 From: SzabolcsGergely Date: Wed, 5 Jul 2023 14:41:00 +0300 Subject: [PATCH 21/87] Update core --- depthai-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/depthai-core b/depthai-core index b4fe512ee..4faa36e13 160000 --- a/depthai-core +++ b/depthai-core @@ -1 +1 @@ -Subproject commit b4fe512eeed011ff8c3438a5c5cee85ac3418471 +Subproject commit 4faa36e1314958285b221430da4e2a639105b480 From 0efec257631f11a5d334d2f37ff3468656372ebf Mon Sep 17 00:00:00 2001 From: SzabolcsGergely Date: Wed, 5 Jul 2023 15:39:17 +0300 Subject: [PATCH 22/87] Update rgb-depth aligned example to be more generic; support alpha scaling parameter --- depthai-core | 2 +- examples/StereoDepth/rgb_depth_aligned.py | 29 +++++++++++++++-------- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/depthai-core b/depthai-core index 4faa36e13..945e7b8ce 160000 --- a/depthai-core +++ b/depthai-core @@ -1 +1 @@ -Subproject commit 4faa36e1314958285b221430da4e2a639105b480 +Subproject commit 945e7b8ceaf61eda0dbc982faa8208d7bca9d0b0 diff --git a/examples/StereoDepth/rgb_depth_aligned.py b/examples/StereoDepth/rgb_depth_aligned.py index 3e72fc023..09e2389f0 100755 --- a/examples/StereoDepth/rgb_depth_aligned.py +++ b/examples/StereoDepth/rgb_depth_aligned.py @@ -3,11 +3,16 @@ import cv2 import numpy as np import depthai as dai +import argparse # Weights to use when blending depth/rgb image (should equal 1.0) rgbWeight = 0.4 depthWeight = 0.6 +parser = argparse.ArgumentParser() +parser.add_argument('-alpha', type=float, default=None, help="Alpha scaling parameter to increase float. [0,1] valid interval.") +args = parser.parse_args() +alpha = args.alpha def updateBlendWeights(percent_rgb): """ @@ -21,9 +26,6 @@ def updateBlendWeights(percent_rgb): depthWeight = 1.0 - rgbWeight -# Optional. If set (True), the ColorCamera is downscaled from 1080p to 720p. -# Otherwise (False), the aligned depth is automatically upscaled to 1080p -downscaleColor = True fps = 30 # The disparity is computed at this resolution, then upscaled to RGB resolution monoResolution = dai.MonoCameraProperties.SensorResolution.THE_720_P @@ -34,7 +36,7 @@ def updateBlendWeights(percent_rgb): queueNames = [] # Define sources and outputs -camRgb = pipeline.create(dai.node.ColorCamera) +camRgb = pipeline.create(dai.node.Camera) left = pipeline.create(dai.node.MonoCamera) right = pipeline.create(dai.node.MonoCamera) stereo = pipeline.create(dai.node.StereoDepth) @@ -48,15 +50,17 @@ def updateBlendWeights(percent_rgb): queueNames.append("disp") #Properties -camRgb.setBoardSocket(dai.CameraBoardSocket.CAM_A) -camRgb.setResolution(dai.ColorCameraProperties.SensorResolution.THE_1080_P) +rgbCamSocket = dai.CameraBoardSocket.CAM_A + +camRgb.setBoardSocket(rgbCamSocket) +camRgb.setSize(1280, 720) camRgb.setFps(fps) -if downscaleColor: camRgb.setIspScale(2, 3) + # For now, RGB needs fixed focus to properly align with depth. # This value was used during calibration try: calibData = device.readCalibration2() - lensPosition = calibData.getLensPosition(dai.CameraBoardSocket.CAM_A) + lensPosition = calibData.getLensPosition(rgbCamSocket) if lensPosition: camRgb.initialControl.setManualFocus(lensPosition) except: @@ -71,14 +75,19 @@ def updateBlendWeights(percent_rgb): stereo.setDefaultProfilePreset(dai.node.StereoDepth.PresetMode.HIGH_DENSITY) # LR-check is required for depth alignment stereo.setLeftRightCheck(True) -stereo.setDepthAlign(dai.CameraBoardSocket.CAM_A) +stereo.setDepthAlign(rgbCamSocket) # Linking -camRgb.isp.link(rgbOut.input) +camRgb.video.link(rgbOut.input) left.out.link(stereo.left) right.out.link(stereo.right) stereo.disparity.link(disparityOut.input) +camRgb.setMeshSource(dai.CameraProperties.WarpMeshSource.CALIBRATION) +if alpha is not None: + camRgb.setCalibrationAlpha(alpha) + stereo.setAlphaScaling(alpha) + # Connect to device and start pipeline with device: device.startPipeline(pipeline) From 10b598e95b132533d003e96eda6c7b05eeaf2ff5 Mon Sep 17 00:00:00 2001 From: SzabolcsGergely Date: Wed, 5 Jul 2023 20:24:47 +0300 Subject: [PATCH 23/87] BMI: Fix accumulating latency due to slow xlink read --- depthai-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/depthai-core b/depthai-core index 945e7b8ce..ca3de8621 160000 --- a/depthai-core +++ b/depthai-core @@ -1 +1 @@ -Subproject commit 945e7b8ceaf61eda0dbc982faa8208d7bca9d0b0 +Subproject commit ca3de8621761d91489f43e92bd042a70237c6823 From a5e40a2443d4e2ab9828d5c52758236347d8c180 Mon Sep 17 00:00:00 2001 From: Erol444 Date: Fri, 7 Jul 2023 12:38:08 +0200 Subject: [PATCH 24/87] Added example that undistorts wide fov color cam --- examples/Camera/camera_undistort.py | 32 +++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 examples/Camera/camera_undistort.py diff --git a/examples/Camera/camera_undistort.py b/examples/Camera/camera_undistort.py new file mode 100644 index 000000000..0d78f1d9e --- /dev/null +++ b/examples/Camera/camera_undistort.py @@ -0,0 +1,32 @@ +import depthai as dai +import cv2 + +pipeline = dai.Pipeline() + +# Define sources and outputs +camRgb: dai.node.Camera = pipeline.create(dai.node.Camera) + +#Properties +camRgb.setBoardSocket(dai.CameraBoardSocket.CAM_A) +camRgb.setSize((1280, 800)) + +# Linking +videoOut = pipeline.create(dai.node.XLinkOut) +videoOut.setStreamName("video") +camRgb.video.link(videoOut.input) + +ispOut = pipeline.create(dai.node.XLinkOut) +ispOut.setStreamName("isp") +camRgb.isp.link(ispOut.input) + +with dai.Device(pipeline) as device: + video = device.getOutputQueue(name="video", maxSize=1, blocking=False) + isp = device.getOutputQueue(name="isp", maxSize=1, blocking=False) + + while True: + if video.has(): + cv2.imshow("video", video.get().getCvFrame()) + if isp.has(): + cv2.imshow("isp", isp.get().getCvFrame()) + if cv2.waitKey(1) == ord('q'): + break From 4c88cc86a31b2020b54bf09b2d69673422284bd3 Mon Sep 17 00:00:00 2001 From: Erol444 Date: Fri, 7 Jul 2023 12:45:19 +0200 Subject: [PATCH 25/87] Added example docs --- .../samples/Camera/camera_undistort.rst | 43 +++++++++++++++++++ docs/source/tutorials/code_samples.rst | 5 +++ 2 files changed, 48 insertions(+) create mode 100644 docs/source/samples/Camera/camera_undistort.rst diff --git a/docs/source/samples/Camera/camera_undistort.rst b/docs/source/samples/Camera/camera_undistort.rst new file mode 100644 index 000000000..eaa43da43 --- /dev/null +++ b/docs/source/samples/Camera/camera_undistort.rst @@ -0,0 +1,43 @@ +Undistort camera stream +======================= + +This example shows how you can use :ref:`Camera` node to undistort wide FOV camera stream. :ref:`Camera` node will automatically undistort ``still``, ``video`` and ``preview`` streams, while ``isp`` stream will be left as is. + +Demo +#### + +.. figure:: https://github.com/luxonis/depthai-python/assets/18037362/936b9ad7-179b-42a5-a6cb-25efdbdf73d9 + + Left: Camera.isp output. Right: Camera.video (undistorted) output + +Setup +##### + +.. include:: /includes/install_from_pypi.rst + +Source code +########### + +.. tabs:: + + .. tab:: Python + + Also `available on GitHub `__ + + .. literalinclude:: ../../../../examples/Camera/camera_undistort.py + :language: python + :linenos: + + .. tab:: C++ + + Work in progress. + + +.. + Also `available on GitHub `__ + + .. literalinclude:: ../../../../depthai-core/examples/Camera/camera_undistort.cpp + :language: cpp + :linenos: + +.. include:: /includes/footer-short.rst diff --git a/docs/source/tutorials/code_samples.rst b/docs/source/tutorials/code_samples.rst index 8d1af2a57..603d46893 100644 --- a/docs/source/tutorials/code_samples.rst +++ b/docs/source/tutorials/code_samples.rst @@ -44,6 +44,11 @@ are presented with code. - :ref:`Calibration Reader` - Reads calibration data stored on device over XLink - :ref:`Calibration Load` - Loads and uses calibration data of version 6 (gen2 calibration data) in a pipeline + +.. rubric:: Camera + +- :ref:`Undistort camera stream` - Showcases how Camera node undistorts camera streams + .. rubric:: ColorCamera - :ref:`Auto Exposure on ROI` - Demonstrates how to use auto exposure based on the selected ROI From 15e386345b1ed6df9ca0cdc3a076a975a01b0232 Mon Sep 17 00:00:00 2001 From: Erol444 Date: Wed, 12 Jul 2023 14:09:32 +0200 Subject: [PATCH 26/87] Updated cameraProperties bindings --- src/pipeline/node/CameraBindings.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/pipeline/node/CameraBindings.cpp b/src/pipeline/node/CameraBindings.cpp index 2c52bb282..f8adc2922 100644 --- a/src/pipeline/node/CameraBindings.cpp +++ b/src/pipeline/node/CameraBindings.cpp @@ -60,23 +60,28 @@ void bind_camera(pybind11::module& m, void* pCallstack){ cameraProperties .def_readwrite("initialControl", &CameraProperties::initialControl) .def_readwrite("boardSocket", &CameraProperties::boardSocket) + .def_readwrite("cameraName", &CameraProperties::cameraName) .def_readwrite("imageOrientation", &CameraProperties::imageOrientation) .def_readwrite("colorOrder", &CameraProperties::colorOrder) .def_readwrite("interleaved", &CameraProperties::interleaved) .def_readwrite("fp16", &CameraProperties::fp16) + .def_readwrite("previewHeight", &CameraProperties::previewHeight) .def_readwrite("previewWidth", &CameraProperties::previewWidth) .def_readwrite("videoHeight", &CameraProperties::videoHeight) .def_readwrite("videoWidth", &CameraProperties::videoWidth) .def_readwrite("stillHeight", &CameraProperties::stillHeight) .def_readwrite("stillWidth", &CameraProperties::stillWidth) - // .def_readwrite("resolution", &CameraProperties::resolution) + .def_readwrite("resolutionHeight", &CameraProperties::resolutionHeight) + .def_readwrite("resolutionWidth", &CameraProperties::resolutionWidth) + .def_readwrite("fps", &CameraProperties::fps) .def_readwrite("isp3aFps", &CameraProperties::isp3aFps) .def_readwrite("sensorCropX", &CameraProperties::sensorCropX) .def_readwrite("sensorCropY", &CameraProperties::sensorCropY) .def_readwrite("previewKeepAspectRatio", &CameraProperties::previewKeepAspectRatio) .def_readwrite("ispScale", &CameraProperties::ispScale) + .def_readwrite("sensorType", &CameraProperties::sensorType) .def_readwrite("numFramesPoolRaw", &CameraProperties::numFramesPoolRaw) .def_readwrite("numFramesPoolIsp", &CameraProperties::numFramesPoolIsp) @@ -91,6 +96,7 @@ void bind_camera(pybind11::module& m, void* pCallstack){ .def_readwrite("calibAlpha", &CameraProperties::calibAlpha) .def_readwrite("warpMeshStepWidth", &CameraProperties::warpMeshStepWidth) .def_readwrite("warpMeshStepHeight", &CameraProperties::warpMeshStepHeight) + .def_readwrite("rawPacked", &CameraProperties::rawPacked) ; // Camera node From 30e01928f8627f64223d525fb4e3c0fe52e649e2 Mon Sep 17 00:00:00 2001 From: Erol444 Date: Tue, 18 Jul 2023 12:08:17 +0200 Subject: [PATCH 27/87] Added camera bindings for linting --- src/pipeline/PipelineBindings.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pipeline/PipelineBindings.cpp b/src/pipeline/PipelineBindings.cpp index 3b4248710..839944b9f 100644 --- a/src/pipeline/PipelineBindings.cpp +++ b/src/pipeline/PipelineBindings.cpp @@ -144,6 +144,7 @@ void PipelineBindings::bind(pybind11::module& m, void* pCallstack){ .def("createAprilTag", &Pipeline::create) .def("createDetectionParser", &Pipeline::create) .def("createUVC", &Pipeline::create) + .def("createCamera", &Pipeline::create) ; From 963760827fc70a64c3e0a96a01729654906b2e4a Mon Sep 17 00:00:00 2001 From: Rupesh Jeyaram Date: Mon, 24 Jul 2023 12:07:24 -0400 Subject: [PATCH 28/87] Update imu.rst Just updating a small typo in a readme! (intertial -> inertial) --- docs/source/components/nodes/imu.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/components/nodes/imu.rst b/docs/source/components/nodes/imu.rst index 3859ecd67..9489856d3 100644 --- a/docs/source/components/nodes/imu.rst +++ b/docs/source/components/nodes/imu.rst @@ -1,7 +1,7 @@ IMU === -IMU (`intertial measurement unit `__) node can be used to receive data +IMU (`inertial measurement unit `__) node can be used to receive data from the IMU chip on the device. Our OAK devices use either: - `BNO085 `__ (`datasheet here `__) 9-axis sensor, combining accelerometer, gyroscope, and magnetometer. It also does sensor fusion on the (IMU) chip itself. We have efficiently integrated `this driver `__ into the DepthAI. From 01f81e09ffedbc3b86bfc44006fb3ff5d81adc72 Mon Sep 17 00:00:00 2001 From: Erol444 Date: Tue, 25 Jul 2023 16:12:35 +0200 Subject: [PATCH 29/87] Added camera node docs --- docs/source/components/nodes/camera.rst | 149 ++++++++++++++++++++++++ 1 file changed, 149 insertions(+) create mode 100644 docs/source/components/nodes/camera.rst diff --git a/docs/source/components/nodes/camera.rst b/docs/source/components/nodes/camera.rst new file mode 100644 index 000000000..997439c49 --- /dev/null +++ b/docs/source/components/nodes/camera.rst @@ -0,0 +1,149 @@ +Camera +====== + +Camera node is a source of :ref:`image frames `. You can control in at runtime with the :code:`InputControl` and :code:`InputConfig`. +It aims to unify the :ref:`Camera` and :ref:`MonoCamera` into one node. + +Compared to :ref:`ColorCamera`, Camera node: + +- Supports **cam.setSize()**, which replaces both ``cam.setResolution()`` and ``cam.setIspScale()``. Camera node will automatically find resolution that fits best, and apply correct scaling to achieve user-selected size +- Supports **cam.setCalibrationAlpha()**, example here: :ref:`Undistort camera stream` +- Supports **cam.loadMeshData()** and **cam.setMeshStep()**, which can be used for custom image warping (undistortion, perspective correction, etc.) + +Besides points above, compared to :ref:`MonoCamera` Camera node: + +- Doesn't have ``out`` output, but same outputs as :ref:`ColorCamera` (``raw``, ``isp``, ``still``, ``preview``, ``video``). This means that ``preview`` will output 3 planes of the same grayscale frame (3x overhead), and ``isp`` / ``video`` / ``still`` will output luma (useful grayscale information) + chroma (all values are 128), which will result in 1.5x bandwidth overhead + +How to place it +############### + +.. tabs:: + + .. code-tab:: py + + pipeline = dai.Pipeline() + cam = pipeline.create(dai.node.Camera) + + .. code-tab:: c++ + + dai::Pipeline pipeline; + auto cam = pipeline.create(); + + +Inputs and Outputs +################## + +.. code-block:: + + Camera node + ┌──────────────────────────────┐ + │ ┌─────────────┐ │ + │ │ Image │ raw │ raw + │ │ Sensor │---┬--------├────────► + │ └────▲────────┘ | │ + │ │ ┌--------┘ │ + │ ┌─┴───▼─┐ │ isp + inputControl │ │ │-------┬-------├────────► + ──────────────►│------│ ISP │ ┌─────▼────┐ │ video + │ │ │ | |--├────────► + │ └───────┘ │ Image │ │ still + inputConfig │ │ Post- │--├────────► + ──────────────►│----------------|Processing│ │ preview + │ │ │--├────────► + │ └──────────┘ │ + └──────────────────────────────┘ + +**Message types** + +- :code:`inputConfig` - :ref:`ImageManipConfig` +- :code:`inputControl` - :ref:`CameraControl` +- :code:`raw` - :ref:`ImgFrame` - RAW10 bayer data. Demo code for unpacking `here `__ +- :code:`isp` - :ref:`ImgFrame` - YUV420 planar (same as YU12/IYUV/I420) +- :code:`still` - :ref:`ImgFrame` - NV12, suitable for bigger size frames. The image gets created when a capture event is sent to the Camera, so it's like taking a photo +- :code:`preview` - :ref:`ImgFrame` - RGB (or BGR planar/interleaved if configured), mostly suited for small size previews and to feed the image into :ref:`NeuralNetwork` +- :code:`video` - :ref:`ImgFrame` - NV12, suitable for bigger size frames + +**ISP** (image signal processor) is used for bayer transformation, demosaicing, noise reduction, and other image enhancements. +It interacts with the 3A algorithms: **auto-focus**, **auto-exposure**, and **auto-white-balance**, which are handling image sensor +adjustments such as exposure time, sensitivity (ISO), and lens position (if the camera module has a motorized lens) at runtime. +Click `here `__ for more information. + +**Image Post-Processing** converts YUV420 planar frames from the **ISP** into :code:`video`/:code:`preview`/:code:`still` frames. + +``still`` (when a capture is triggered) and ``isp`` work at the max camera resolution, while ``video`` and ``preview`` are +limited to max 4K (3840 x 2160) resolution, which is cropped from ``isp``. +For IMX378 (12MP), the **post-processing** works like this: + +.. code-block:: + + ┌─────┐ Cropping to ┌─────────┐ Downscaling ┌──────────┐ + │ ISP ├────────────────►│ video ├───────────────►│ preview │ + └─────┘ max 3840x2160 └─────────┘ and cropping └──────────┘ + +.. image:: /_static/images/tutorials/isp.jpg + +The image above is the ``isp`` output from the Camera (12MP resolution from IMX378). If you aren't downscaling ISP, +the ``video`` output is cropped to 4k (max 3840x2160 due to the limitation of the ``video`` output) as represented by +the blue rectangle. The Yellow rectangle represents a cropped ``preview`` output when the preview size is set to a 1:1 aspect +ratio (eg. when using a 300x300 preview size for the MobileNet-SSD NN model) because the ``preview`` output is derived from +the ``video`` output. + +Usage +##### + +.. tabs:: + + .. code-tab:: py + + pipeline = dai.Pipeline() + cam = pipeline.create(dai.node.Camera) + cam.setPreviewSize(300, 300) + cam.setBoardSocket(dai.CameraBoardSocket.CAM_A) + # Instead of setting the resolution, user can specify size, which will set + # sensor resolution to best fit, and also apply scaling + cam.setSize(1280, 720) + + .. code-tab:: c++ + + dai::Pipeline pipeline; + auto cam = pipeline.create(); + cam->setPreviewSize(300, 300); + cam->setBoardSocket(dai::CameraBoardSocket::CAM_A); + // Instead of setting the resolution, user can specify size, which will set + // sensor resolution to best fit, and also apply scaling + cam->setSize(1280, 720); + +Limitations +########### + +Here are known camera limitations for the `RVC2 `__: + +- **ISP can process about 600 MP/s**, and about **500 MP/s** when the pipeline is also running NNs and video encoder in parallel +- **3A algorithms** can process about **200..250 FPS overall** (for all camera streams). This is a current limitation of our implementation, and we have plans for a workaround to run 3A algorithms on every Xth frame, no ETA yet + +Examples of functionality +######################### + +- :ref:`Undistort camera stream` + +Reference +######### + +.. tabs:: + + .. tab:: Python + + .. autoclass:: depthai.node.Camera + :members: + :inherited-members: + :noindex: + + .. tab:: C++ + + .. doxygenclass:: dai::node::Camera + :project: depthai-core + :members: + :private-members: + :undoc-members: + +.. include:: ../../includes/footer-short.rst From cddd62261e7319602c28023ff67422b8f24e35ed Mon Sep 17 00:00:00 2001 From: Erol444 Date: Tue, 25 Jul 2023 17:46:08 +0200 Subject: [PATCH 30/87] Fix docs building --- docs/source/tutorials/code_samples.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/tutorials/code_samples.rst b/docs/source/tutorials/code_samples.rst index 603d46893..a5d9f3e2f 100644 --- a/docs/source/tutorials/code_samples.rst +++ b/docs/source/tutorials/code_samples.rst @@ -7,6 +7,7 @@ Code Samples ../samples/bootloader/* ../samples/calibration/* + ../samples/Camera/* ../samples/ColorCamera/* ../samples/crash_report/* ../samples/EdgeDetector/* From f325d49e24202588079f349116fad7f78fb6ac31 Mon Sep 17 00:00:00 2001 From: Erol444 Date: Tue, 25 Jul 2023 18:26:22 +0200 Subject: [PATCH 31/87] Update docs/source/components/nodes/camera.rst Co-authored-by: TheMarpe --- docs/source/components/nodes/camera.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/components/nodes/camera.rst b/docs/source/components/nodes/camera.rst index 997439c49..42f5543a1 100644 --- a/docs/source/components/nodes/camera.rst +++ b/docs/source/components/nodes/camera.rst @@ -2,7 +2,7 @@ Camera ====== Camera node is a source of :ref:`image frames `. You can control in at runtime with the :code:`InputControl` and :code:`InputConfig`. -It aims to unify the :ref:`Camera` and :ref:`MonoCamera` into one node. +It aims to unify the :ref:`ColorCamera` and :ref:`MonoCamera` into one node. Compared to :ref:`ColorCamera`, Camera node: From c70426358c3f7b09efc65118acc23c3144928cb0 Mon Sep 17 00:00:00 2001 From: Erol444 Date: Tue, 25 Jul 2023 19:44:51 +0200 Subject: [PATCH 32/87] Fix typos --- docs/source/components/nodes/camera.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/components/nodes/camera.rst b/docs/source/components/nodes/camera.rst index 42f5543a1..32740baa8 100644 --- a/docs/source/components/nodes/camera.rst +++ b/docs/source/components/nodes/camera.rst @@ -4,15 +4,15 @@ Camera Camera node is a source of :ref:`image frames `. You can control in at runtime with the :code:`InputControl` and :code:`InputConfig`. It aims to unify the :ref:`ColorCamera` and :ref:`MonoCamera` into one node. -Compared to :ref:`ColorCamera`, Camera node: +Compared to :ref:`ColorCamera` node, Camera node: - Supports **cam.setSize()**, which replaces both ``cam.setResolution()`` and ``cam.setIspScale()``. Camera node will automatically find resolution that fits best, and apply correct scaling to achieve user-selected size - Supports **cam.setCalibrationAlpha()**, example here: :ref:`Undistort camera stream` - Supports **cam.loadMeshData()** and **cam.setMeshStep()**, which can be used for custom image warping (undistortion, perspective correction, etc.) -Besides points above, compared to :ref:`MonoCamera` Camera node: +Besides points above, compared to :ref:`MonoCamera` node, Camera node: -- Doesn't have ``out`` output, but same outputs as :ref:`ColorCamera` (``raw``, ``isp``, ``still``, ``preview``, ``video``). This means that ``preview`` will output 3 planes of the same grayscale frame (3x overhead), and ``isp`` / ``video`` / ``still`` will output luma (useful grayscale information) + chroma (all values are 128), which will result in 1.5x bandwidth overhead +- Doesn't have ``out`` output, as it has the same outputs as :ref:`ColorCamera` (``raw``, ``isp``, ``still``, ``preview``, ``video``). This means that ``preview`` will output 3 planes of the same grayscale frame (3x overhead), and ``isp`` / ``video`` / ``still`` will output luma (useful grayscale information) + chroma (all values are 128), which will result in 1.5x bandwidth overhead How to place it ############### From 236e7ec1b8ac2c7573d980f45efe90e9b3df812f Mon Sep 17 00:00:00 2001 From: TheMarpe Date: Wed, 26 Jul 2023 00:48:26 +0200 Subject: [PATCH 33/87] Updated wording on 'install_depthai.sh' script --- docs/source/_static/install_depthai.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/source/_static/install_depthai.sh b/docs/source/_static/install_depthai.sh index 27aa3b75c..e36ee888a 100755 --- a/docs/source/_static/install_depthai.sh +++ b/docs/source/_static/install_depthai.sh @@ -53,7 +53,7 @@ do if [[ "$python_version" == "" ]]; then echo "No python version found." echo "Input path for python binary, version 3.8 or higher, or leave empty and python 3.10 will be installed for you." - echo "Press any key to continue" + echo "Press ENTER key to continue" read -e python_binary_path < /dev/tty # python not found and user wants to install python 3.10 if [ "$python_binary_path" = "" ]; then @@ -66,8 +66,8 @@ do nr_2=$(echo "${python_version:9:2}" | tr -d -c 0-9) echo "Python version: $python_version found." if [ "$nr_1" -gt 2 ] && [ "$nr_2" -gt 7 ]; then # first two digits of python version greater then 3.7 -> python version 3.8 or greater is allowed. - echo "If you want to use it for installation, press ANY key, otherwise input path to python binary." - echo "Press any key to continue" + echo "If you want to use it for installation, press ENTER key, otherwise input path to python binary." + echo "Press ENTER key to continue" read -e python_binary_path < /dev/tty # user wants to use already installed python whose version is high enough if [ "$python_binary_path" = "" ]; then @@ -75,7 +75,7 @@ do fi else echo "This python version is not supported by depthai. Enter path to python binary version et least 3.8, or leave empty and python 3.10 will be installed automatically." - echo "Press any key to continue" + echo "Press ENTER key to continue" read -e python_binary_path < /dev/tty # python version is too low and user wants to install python 3.10 if [ "$python_binary_path" = "" ]; then @@ -241,7 +241,7 @@ fi echo -e '\n\n:::::::::::::::: INSTALATION COMPLETE ::::::::::::::::\n' echo -e '\nTo run demo app write in terminal.' -echo "Press ANY KEY to finish and run the demo app..." +echo "Press ENTER KEY to finish and run the demo app..." read -n1 key < /dev/tty echo "STARTING DEMO APP." python "$DEPTHAI_DIR/launcher/launcher.py" -r "$DEPTHAI_DIR" From a5f2340c585fca9099de464bf0c48a3521646058 Mon Sep 17 00:00:00 2001 From: Erol444 Date: Wed, 26 Jul 2023 12:22:53 +0200 Subject: [PATCH 34/87] Update disparity_colormap_encoding.py --- examples/VideoEncoder/disparity_colormap_encoding.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/VideoEncoder/disparity_colormap_encoding.py b/examples/VideoEncoder/disparity_colormap_encoding.py index c0d2c8ce1..fbaf27fb2 100755 --- a/examples/VideoEncoder/disparity_colormap_encoding.py +++ b/examples/VideoEncoder/disparity_colormap_encoding.py @@ -59,4 +59,4 @@ pass print("To view the encoded data, convert the stream file (.mjpeg) into a video file (.mp4) using a command below:") - print("ffmpeg -framerate 30 -i disparity.mjpeg -c copy video.mp4") + print("ffmpeg -framerate 30 -i disparity.h264 -c copy video.mp4") From cb7243e5c5c37461b1ac966de69ca6c8889b7522 Mon Sep 17 00:00:00 2001 From: Martin Peterlin Date: Thu, 27 Jul 2023 18:59:03 +0200 Subject: [PATCH 35/87] Added getProductName and tweaked getDeviceName --- depthai-core | 2 +- src/CalibrationHandlerBindings.cpp | 3 +++ src/DeviceBindings.cpp | 1 + src/pipeline/CommonBindings.cpp | 7 ++++--- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/depthai-core b/depthai-core index c7f3076ab..dbed22859 160000 --- a/depthai-core +++ b/depthai-core @@ -1 +1 @@ -Subproject commit c7f3076abb1ebdbb1abf0ca34c14361635c703cf +Subproject commit dbed228593e201bc8c7d28ab24b967c32383cab6 diff --git a/src/CalibrationHandlerBindings.cpp b/src/CalibrationHandlerBindings.cpp index 52f7bf3ae..69abd737a 100644 --- a/src/CalibrationHandlerBindings.cpp +++ b/src/CalibrationHandlerBindings.cpp @@ -62,6 +62,9 @@ void CalibrationHandlerBindings::bind(pybind11::module& m, void* pCallstack){ .def("setBoardInfo", py::overload_cast(&CalibrationHandler::setBoardInfo), py::arg("boardName"), py::arg("boardRev"), DOC(dai, CalibrationHandler, setBoardInfo)) .def("setBoardInfo", py::overload_cast(&CalibrationHandler::setBoardInfo), py::arg("productName"), py::arg("boardName"), py::arg("boardRev"), py::arg("boardConf"), py::arg("hardwareConf"), py::arg("batchName"), py::arg("batchTime"), py::arg("boardOptions"), py::arg("boardCustom") = "", DOC(dai, CalibrationHandler, setBoardInfo, 2)) + .def("setBoardInfo", py::overload_cast(&CalibrationHandler::setBoardInfo), py::arg("deviceName"), py::arg("productName"), py::arg("boardName"), py::arg("boardRev"), py::arg("boardConf"), py::arg("hardwareConf"), py::arg("batchName"), py::arg("batchTime"), py::arg("boardOptions"), py::arg("boardCustom") = "", DOC(dai, CalibrationHandler, setBoardInfo, 3)) + + .def("setDeviceName", &CalibrationHandler::setDeviceName, py::arg("deviceName"), DOC(dai, CalibrationHandler, setDeviceName)) .def("setProductName", &CalibrationHandler::setProductName, py::arg("productName"), DOC(dai, CalibrationHandler, setProductName)) .def("setCameraIntrinsics", py::overload_cast>, Size2f>(&CalibrationHandler::setCameraIntrinsics), py::arg("cameraId"), py::arg("intrinsics"), py::arg("frameSize"), DOC(dai, CalibrationHandler, setCameraIntrinsics)) diff --git a/src/DeviceBindings.cpp b/src/DeviceBindings.cpp index afd3c99eb..447ab9f81 100644 --- a/src/DeviceBindings.cpp +++ b/src/DeviceBindings.cpp @@ -653,6 +653,7 @@ void DeviceBindings::bind(pybind11::module& m, void* pCallstack){ .def("setTimesync", [](DeviceBase& d, std::chrono::milliseconds p, int s, bool r) { py::gil_scoped_release release; return d.setTimesync(p,s,r); }, DOC(dai, DeviceBase, setTimesync)) .def("setTimesync", [](DeviceBase& d, bool e) { py::gil_scoped_release release; return d.setTimesync(e); }, py::arg("enable"), DOC(dai, DeviceBase, setTimesync, 2)) .def("getDeviceName", [](DeviceBase& d) { std::string name; { py::gil_scoped_release release; name = d.getDeviceName(); } return py::bytes(name).attr("decode")("utf-8", "replace"); }, DOC(dai, DeviceBase, getDeviceName)) + .def("getProductName", [](DeviceBase& d) { std::string name; { py::gil_scoped_release release; name = d.getProductName(); } return py::bytes(name).attr("decode")("utf-8", "replace"); }, DOC(dai, DeviceBase, getProductName)) ; diff --git a/src/pipeline/CommonBindings.cpp b/src/pipeline/CommonBindings.cpp index 8d5f0d50d..b7d278eb3 100644 --- a/src/pipeline/CommonBindings.cpp +++ b/src/pipeline/CommonBindings.cpp @@ -64,7 +64,7 @@ void CommonBindings::bind(pybind11::module& m, void* pCallstack){ py::enum_ frameEvent(m, "FrameEvent", DOC(dai, FrameEvent)); py::class_ profilingData(m, "ProfilingData", DOC(dai, ProfilingData)); py::enum_ interpolation(m, "Interpolation", DOC(dai, Interpolation)); - + /////////////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////////// @@ -294,7 +294,8 @@ void CommonBindings::bind(pybind11::module& m, void* pCallstack){ .def_readwrite("boardConf", &EepromData::boardConf) .def_readwrite("hardwareConf", &EepromData::hardwareConf) .def_readwrite("productName", &EepromData::productName) - .def_readwrite("batchName", &EepromData::batchName) + // .def_readwrite("batchName", &EepromData::batchName) + .def_readwrite("deviceName", &EepromData::deviceName) .def_readwrite("batchTime", &EepromData::batchTime) .def_readwrite("boardOptions", &EepromData::boardOptions) .def_readwrite("cameraData", &EepromData::cameraData) @@ -378,7 +379,7 @@ void CommonBindings::bind(pybind11::module& m, void* pCallstack){ .value("READOUT_START", FrameEvent::READOUT_START) .value("READOUT_END", FrameEvent::READOUT_END) ; - + interpolation .value("BILINEAR", Interpolation::BILINEAR) .value("BICUBIC", Interpolation::BICUBIC) From 93bdcbb149f6222c783b67d18b46464667774172 Mon Sep 17 00:00:00 2001 From: TheMarpe Date: Mon, 31 Jul 2023 14:37:47 +0200 Subject: [PATCH 36/87] Fixed device related logging --- depthai-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/depthai-core b/depthai-core index c7f3076ab..0e37dcde9 160000 --- a/depthai-core +++ b/depthai-core @@ -1 +1 @@ -Subproject commit c7f3076abb1ebdbb1abf0ca34c14361635c703cf +Subproject commit 0e37dcde91afc4c633da40eb47887ec266314733 From 2b43e6f1326d3a130421ee0be4e08883c30cd0e7 Mon Sep 17 00:00:00 2001 From: Erol444 Date: Mon, 31 Jul 2023 18:21:19 +0300 Subject: [PATCH 37/87] Updated videoEnc docs --- docs/source/components/nodes/video_encoder.rst | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/source/components/nodes/video_encoder.rst b/docs/source/components/nodes/video_encoder.rst index 4f899de7a..2c3224219 100644 --- a/docs/source/components/nodes/video_encoder.rst +++ b/docs/source/components/nodes/video_encoder.rst @@ -81,13 +81,11 @@ Limitations ########### For **H.264 / H.265 encoding**, we have the following limits: -- **248 million pixels/second** limit for the encoder or 3840x2160 pixels at 30FPS. The resolution and frame rate can be divided into multiple streams - but the sum of all the pixels/second needs to be below 248 million. +- **248 million pixels/second** (4K@30) limit for the encoder. The resolution and frame rate can be divided into multiple streams - but the sum of all the pixels/second needs to be below 248 million. - Due to a HW constraint, video encoding can be done only on frames whose width values are multiples of 32. - 4096 pixel max width for a frame. -- Maximum of 3 parallel encoding streams. -The **MJPEG encoder** is capable of 16384x8192 resolution at 500Mpixel/second. From our testing, we were able to encode -4K at 30FPS and 2x 800P at 55FPS. +The **MJPEG encoder** is capable of 16384x8192 resolution at 450 MPix/sec. From our testing, we were able to encode 4K at 30FPS and 2x 800P at 55FPS. Note the processing resources of the encoder **are shared** between H.26x and JPEG. From 197b681979695849377f66dfdb7896f36c4ad782 Mon Sep 17 00:00:00 2001 From: TheMarpe Date: Wed, 2 Aug 2023 15:39:40 +0200 Subject: [PATCH 38/87] Fixed armv6l&armv7l wheels --- .github/workflows/main.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6a5fc8563..02343505d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -154,17 +154,17 @@ jobs: run: echo "BUILD_COMMIT_HASH=${{github.sha}}" >> $GITHUB_ENV - name: Building wheel run: python3 -m pip wheel . -w ./wheelhouse/ --verbose - - name: Auditing wheel - run: for whl in wheelhouse/*.whl; do auditwheel repair "$whl" --plat linux_armv7l -w wheelhouse/audited/; done - - name: Install tweaked auditwheel and add armv6l tag + - name: Auditing wheels and adding armv6l tag (Running on RPi, binaries compiled as armv6l) run: | - python3 -m pip install git+https://github.com/luxonis/auditwheel@main - for whl in wheelhouse/*.whl; do python3 -m auditwheel addtag -t linux_armv7l linux_armv6l -w wheelhouse/postaudited/ "$whl"; done + for whl in wheelhouse/*.whl; do auditwheel repair "$whl" --plat linux_armv7l -w wheelhouse/preaudited/; done + for whl in wheelhouse/preaudited/*.whl; do python3 -m wheel tags --platform-tag +linux_armv6l "$whl"; done + mkdir -p wheelhouse/audited/ + for whl in wheelhouse/preaudited/*linux_armv6l*.whl; do cp "$whl" wheelhouse/audited/$(basename $whl); done - name: Archive wheel artifacts uses: actions/upload-artifact@v3 with: name: audited-wheels - path: wheelhouse/postaudited/ + path: wheelhouse/audited/ - name: Deploy wheels to artifactory (if not a release) if: startsWith(github.ref, 'refs/tags/v') != true run: bash ./ci/upload-artifactory.sh From bc67691862a45f3b633b8bc5e9c3696a6d2584d5 Mon Sep 17 00:00:00 2001 From: TheMarpe Date: Wed, 2 Aug 2023 18:17:40 +0200 Subject: [PATCH 39/87] Add wheel & auditwheel update for CI --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 02343505d..eed6deefa 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -156,6 +156,7 @@ jobs: run: python3 -m pip wheel . -w ./wheelhouse/ --verbose - name: Auditing wheels and adding armv6l tag (Running on RPi, binaries compiled as armv6l) run: | + python3 -m pip install -U wheel auditwheel for whl in wheelhouse/*.whl; do auditwheel repair "$whl" --plat linux_armv7l -w wheelhouse/preaudited/; done for whl in wheelhouse/preaudited/*.whl; do python3 -m wheel tags --platform-tag +linux_armv6l "$whl"; done mkdir -p wheelhouse/audited/ From f5eb3c4a009fda10312aee85dfcf344ceb7d9a00 Mon Sep 17 00:00:00 2001 From: asahtik Date: Mon, 7 Aug 2023 12:25:25 +0200 Subject: [PATCH 40/87] Added stereo evaluation script --- examples/StereoDepth/read_write_pfm.py | 65 ++ .../StereoDepth/stereo_depth_evaluation.py | 918 ++++++++++++++++++ 2 files changed, 983 insertions(+) create mode 100644 examples/StereoDepth/read_write_pfm.py create mode 100644 examples/StereoDepth/stereo_depth_evaluation.py diff --git a/examples/StereoDepth/read_write_pfm.py b/examples/StereoDepth/read_write_pfm.py new file mode 100644 index 000000000..03fc02927 --- /dev/null +++ b/examples/StereoDepth/read_write_pfm.py @@ -0,0 +1,65 @@ +import numpy as np +import re +import sys + +def read_pfm(file): + file = open(file, "rb") + + color = None + width = None + height = None + scale = None + endian = None + + header = file.readline().rstrip() + if header.decode("ascii") == "PF": + color = True + elif header.decode("ascii") == "Pf": + color = False + else: + raise Exception("Not a PFM file.") + + dim_match = re.search(r"(\d+)\s(\d+)", file.readline().decode("ascii")) + if dim_match: + width, height = map(int, dim_match.groups()) + else: + raise Exception("Malformed PFM header.") + + scale = float(file.readline().rstrip()) + if scale < 0: # little-endian + endian = "<" + scale = -scale + else: + endian = ">" # big-endian + + data = np.fromfile(file, endian + "f") + shape = (height, width, 3) if color else (height, width) + return np.flip(np.reshape(data, shape), axis=0), scale + +def write_pfm(file, image, scale = 1. / 255): + image = np.flip(image, axis=0) + file = open(file, "wb") + + color = None + + if image.dtype.name != "float32": + raise Exception("Image dtype must be float32.") + + if len(image.shape) == 3 and image.shape[2] == 3: # color image + color = True + elif len(image.shape) == 2 or len(image.shape) == 3 and image.shape[2] == 1: # greyscale + color = False + else: + raise Exception("Image must have H x W x 3, H x W x 1 or H x W dimensions.") + + file.write(b"PF\n" if color else b"Pf\n") + file.write(b"%d %d\n" % (image.shape[1], image.shape[0])) + + endian = image.dtype.byteorder + + if endian == "<" or endian == "=" and sys.byteorder == "little": + scale = -scale + + file.write(b"%f\n" % scale) + + image.tofile(file) diff --git a/examples/StereoDepth/stereo_depth_evaluation.py b/examples/StereoDepth/stereo_depth_evaluation.py new file mode 100644 index 000000000..bd154a760 --- /dev/null +++ b/examples/StereoDepth/stereo_depth_evaluation.py @@ -0,0 +1,918 @@ +#!/usr/bin/env python3 + +import cv2 +import numpy as np +import depthai as dai +from time import sleep +import datetime +import argparse +from pathlib import Path +import math +import os +from read_write_pfm import read_pfm + +datasetDefault = str((Path(__file__).parent / Path("models/dataset")).resolve().absolute()) +parser = argparse.ArgumentParser() +parser.add_argument("-p", "--dataset", nargs="?", help="Path to dataset", default=datasetDefault) +parser.add_argument("-d", "--debug", action="store_true", help="Enable debug outputs.") +parser.add_argument("-e", "--evaluate", action="store_true", help="Output GT evaluation.") +parser.add_argument("-dumpdispcost", "--dumpdisparitycostvalues", action="store_true", help="Dumps the disparity cost values for each disparity range. 96 byte for each pixel.") +parser.add_argument("--download", action="store_true", help="Downloads the 2014 Middlebury dataset.", default=False) +args = parser.parse_args() + +if not Path(args.dataset).exists(): + if not args.download: + import sys + raise FileNotFoundError(f"Dataset path does not exist, use the --dataset argument to specify the path.") + else: + os.makedirs(args.dataset) + +def download_2014_middlebury(data_path): + import requests, zipfile, io, re + url = "https://vision.middlebury.edu/stereo/data/scenes2014/zip/" + r = requests.get(url) + c = r.content + reg = re.compile(r"href=('|\")(.+\.zip)('|\")") + matches = reg.findall(c.decode("utf-8")) + files = [m[1] for m in matches] + + for f in files: + print(f"Downloading {f} from {url + f}") + r = requests.get(url + f) + print(f"Extracting {f}") + z = zipfile.ZipFile(io.BytesIO(r.content)) + z.extractall(os.path.join(data_path, f[:-4])) + +if args.download: + download_2014_middlebury(args.dataset) + +class StereoConfigHandler: + + class Trackbar: + def __init__(self, trackbarName, windowName, minValue, maxValue, defaultValue, handler): + self.min = minValue + self.max = maxValue + self.windowName = windowName + self.trackbarName = trackbarName + cv2.createTrackbar(trackbarName, windowName, minValue, maxValue, handler) + cv2.setTrackbarPos(trackbarName, windowName, defaultValue) + + def set(self, value): + if value < self.min: + value = self.min + print(f"{self.trackbarName} min value is {self.min}") + if value > self.max: + value = self.max + print(f"{self.trackbarName} max value is {self.max}") + cv2.setTrackbarPos(self.trackbarName, self.windowName, value) + + class CensusMaskHandler: + + stateColor = [(0, 0, 0), (255, 255, 255)] + gridHeight = 50 + gridWidth = 50 + + def fillRectangle(self, row, col): + src = self.gridList[row][col]["topLeft"] + dst = self.gridList[row][col]["bottomRight"] + + stateColor = self.stateColor[1] if self.gridList[row][col]["state"] else self.stateColor[0] + self.changed = True + + cv2.rectangle(self.gridImage, src, dst, stateColor, -1) + cv2.imshow(self.windowName, self.gridImage) + + + def clickCallback(self, event, x, y, flags, param): + if event == cv2.EVENT_LBUTTONDOWN: + col = x * (self.gridSize[1]) // self.width + row = y * (self.gridSize[0]) // self.height + self.gridList[row][col]["state"] = not self.gridList[row][col]["state"] + self.fillRectangle(row, col) + + + def __init__(self, windowName, gridSize): + self.gridSize = gridSize + self.windowName = windowName + self.changed = False + + cv2.namedWindow(self.windowName) + + self.width = StereoConfigHandler.CensusMaskHandler.gridWidth * self.gridSize[1] + self.height = StereoConfigHandler.CensusMaskHandler.gridHeight * self.gridSize[0] + + self.gridImage = np.zeros((self.height + 50, self.width, 3), np.uint8) + + cv2.putText(self.gridImage, "Click on grid to change mask!", (0, self.height+20), cv2.FONT_HERSHEY_SIMPLEX, 0.4, (255, 255, 255)) + cv2.putText(self.gridImage, "White: ON | Black: OFF", (0, self.height+40), cv2.FONT_HERSHEY_SIMPLEX, 0.4, (255, 255, 255)) + + self.gridList = [[dict() for _ in range(self.gridSize[1])] for _ in range(self.gridSize[0])] + + for row in range(self.gridSize[0]): + rowFactor = self.height // self.gridSize[0] + srcY = row*rowFactor + 1 + dstY = (row+1)*rowFactor - 1 + for col in range(self.gridSize[1]): + colFactor = self.width // self.gridSize[1] + srcX = col*colFactor + 1 + dstX = (col+1)*colFactor - 1 + src = (srcX, srcY) + dst = (dstX, dstY) + self.gridList[row][col]["topLeft"] = src + self.gridList[row][col]["bottomRight"] = dst + self.gridList[row][col]["state"] = False + self.fillRectangle(row, col) + + cv2.setMouseCallback(self.windowName, self.clickCallback) + cv2.imshow(self.windowName, self.gridImage) + + def getMask(self) -> np.uint64: + mask = np.uint64(0) + for row in range(self.gridSize[0]): + for col in range(self.gridSize[1]): + if self.gridList[row][col]["state"]: + pos = row*self.gridSize[1] + col + mask = np.bitwise_or(mask, np.uint64(1) << np.uint64(pos)) + + return mask + + def setMask(self, _mask: np.uint64): + mask = np.uint64(_mask) + for row in range(self.gridSize[0]): + for col in range(self.gridSize[1]): + pos = row*self.gridSize[1] + col + if np.bitwise_and(mask, np.uint64(1) << np.uint64(pos)): + self.gridList[row][col]["state"] = True + else: + self.gridList[row][col]["state"] = False + + self.fillRectangle(row, col) + + def isChanged(self): + changed = self.changed + self.changed = False + return changed + + def destroyWindow(self): + cv2.destroyWindow(self.windowName) + + + censusMaskHandler = None + newConfig = False + config = None + trSigma = list() + trConfidence = list() + trLrCheck = list() + trFractionalBits = list() + trLineqAlpha = list() + trLineqBeta = list() + trLineqThreshold = list() + trCostAggregationP1 = list() + trCostAggregationP2 = list() + trTemporalAlpha = list() + trTemporalDelta = list() + trThresholdMinRange = list() + trThresholdMaxRange = list() + trSpeckleRange = list() + trSpatialAlpha = list() + trSpatialDelta = list() + trSpatialHoleFilling = list() + trSpatialNumIterations = list() + trDecimationFactor = list() + trDisparityShift = list() + trCenterAlignmentShift = list() + trInvalidateEdgePixels = list() + + def trackbarSigma(value): + StereoConfigHandler.config.postProcessing.bilateralSigmaValue = value + StereoConfigHandler.newConfig = True + for tr in StereoConfigHandler.trSigma: + tr.set(value) + + def trackbarConfidence(value): + StereoConfigHandler.config.costMatching.confidenceThreshold = value + StereoConfigHandler.newConfig = True + for tr in StereoConfigHandler.trConfidence: + tr.set(value) + + def trackbarLrCheckThreshold(value): + StereoConfigHandler.config.algorithmControl.leftRightCheckThreshold = value + StereoConfigHandler.newConfig = True + for tr in StereoConfigHandler.trLrCheck: + tr.set(value) + + def trackbarFractionalBits(value): + StereoConfigHandler.config.algorithmControl.subpixelFractionalBits = value + StereoConfigHandler.newConfig = True + for tr in StereoConfigHandler.trFractionalBits: + tr.set(value) + + def trackbarLineqAlpha(value): + StereoConfigHandler.config.costMatching.linearEquationParameters.alpha = value + StereoConfigHandler.newConfig = True + for tr in StereoConfigHandler.trLineqAlpha: + tr.set(value) + + def trackbarLineqBeta(value): + StereoConfigHandler.config.costMatching.linearEquationParameters.beta = value + StereoConfigHandler.newConfig = True + for tr in StereoConfigHandler.trLineqBeta: + tr.set(value) + + def trackbarLineqThreshold(value): + StereoConfigHandler.config.costMatching.linearEquationParameters.threshold = value + StereoConfigHandler.newConfig = True + for tr in StereoConfigHandler.trLineqThreshold: + tr.set(value) + + def trackbarCostAggregationP1(value): + StereoConfigHandler.config.costAggregation.horizontalPenaltyCostP1 = value + StereoConfigHandler.config.costAggregation.verticalPenaltyCostP1 = value + StereoConfigHandler.newConfig = True + for tr in StereoConfigHandler.trCostAggregationP1: + tr.set(value) + + def trackbarCostAggregationP2(value): + StereoConfigHandler.config.costAggregation.horizontalPenaltyCostP2 = value + StereoConfigHandler.config.costAggregation.verticalPenaltyCostP2 = value + StereoConfigHandler.newConfig = True + for tr in StereoConfigHandler.trCostAggregationP2: + tr.set(value) + + def trackbarTemporalFilterAlpha(value): + StereoConfigHandler.config.postProcessing.temporalFilter.alpha = value / 100. + StereoConfigHandler.newConfig = True + for tr in StereoConfigHandler.trTemporalAlpha: + tr.set(value) + + def trackbarTemporalFilterDelta(value): + StereoConfigHandler.config.postProcessing.temporalFilter.delta = value + StereoConfigHandler.newConfig = True + for tr in StereoConfigHandler.trTemporalDelta: + tr.set(value) + + def trackbarSpatialFilterAlpha(value): + StereoConfigHandler.config.postProcessing.spatialFilter.alpha = value / 100. + StereoConfigHandler.newConfig = True + for tr in StereoConfigHandler.trSpatialAlpha: + tr.set(value) + + def trackbarSpatialFilterDelta(value): + StereoConfigHandler.config.postProcessing.spatialFilter.delta = value + StereoConfigHandler.newConfig = True + for tr in StereoConfigHandler.trSpatialDelta: + tr.set(value) + + def trackbarSpatialFilterHoleFillingRadius(value): + StereoConfigHandler.config.postProcessing.spatialFilter.holeFillingRadius = value + StereoConfigHandler.newConfig = True + for tr in StereoConfigHandler.trSpatialHoleFilling: + tr.set(value) + + def trackbarSpatialFilterNumIterations(value): + StereoConfigHandler.config.postProcessing.spatialFilter.numIterations = value + StereoConfigHandler.newConfig = True + for tr in StereoConfigHandler.trSpatialNumIterations: + tr.set(value) + + def trackbarThresholdMinRange(value): + StereoConfigHandler.config.postProcessing.thresholdFilter.minRange = value * 1000 + StereoConfigHandler.newConfig = True + for tr in StereoConfigHandler.trThresholdMinRange: + tr.set(value) + + def trackbarThresholdMaxRange(value): + StereoConfigHandler.config.postProcessing.thresholdFilter.maxRange = value * 1000 + StereoConfigHandler.newConfig = True + for tr in StereoConfigHandler.trThresholdMaxRange: + tr.set(value) + + def trackbarSpeckleRange(value): + StereoConfigHandler.config.postProcessing.speckleFilter.speckleRange = value + StereoConfigHandler.newConfig = True + for tr in StereoConfigHandler.trSpeckleRange: + tr.set(value) + + def trackbarDecimationFactor(value): + StereoConfigHandler.config.postProcessing.decimationFilter.decimationFactor = value + StereoConfigHandler.newConfig = True + for tr in StereoConfigHandler.trDecimationFactor: + tr.set(value) + + def trackbarDisparityShift(value): + StereoConfigHandler.config.algorithmControl.disparityShift = value + StereoConfigHandler.newConfig = True + for tr in StereoConfigHandler.trDisparityShift: + tr.set(value) + + def trackbarCenterAlignmentShift(value): + if StereoConfigHandler.config.algorithmControl.depthAlign != dai.StereoDepthConfig.AlgorithmControl.DepthAlign.CENTER: + print("Center alignment shift factor requires CENTER alignment enabled!") + return + StereoConfigHandler.config.algorithmControl.centerAlignmentShiftFactor = value / 100. + print(f"centerAlignmentShiftFactor: {StereoConfigHandler.config.algorithmControl.centerAlignmentShiftFactor:.2f}") + StereoConfigHandler.newConfig = True + for tr in StereoConfigHandler.trCenterAlignmentShift: + tr.set(value) + + def trackbarInvalidateEdgePixels(value): + StereoConfigHandler.config.algorithmControl.numInvalidateEdgePixels = value + print(f"numInvalidateEdgePixels: {StereoConfigHandler.config.algorithmControl.numInvalidateEdgePixels:.2f}") + StereoConfigHandler.newConfig = True + for tr in StereoConfigHandler.trInvalidateEdgePixels: + tr.set(value) + + def handleKeypress(key, stereoDepthConfigInQueue): + if key == ord("m"): + StereoConfigHandler.newConfig = True + medianSettings = [dai.MedianFilter.MEDIAN_OFF, dai.MedianFilter.KERNEL_3x3, dai.MedianFilter.KERNEL_5x5, dai.MedianFilter.KERNEL_7x7] + currentMedian = StereoConfigHandler.config.postProcessing.median + nextMedian = medianSettings[(medianSettings.index(currentMedian)+1) % len(medianSettings)] + print(f"Changing median to {nextMedian.name} from {currentMedian.name}") + StereoConfigHandler.config.postProcessing.median = nextMedian + if key == ord("w"): + StereoConfigHandler.newConfig = True + StereoConfigHandler.config.postProcessing.spatialFilter.enable = not StereoConfigHandler.config.postProcessing.spatialFilter.enable + state = "on" if StereoConfigHandler.config.postProcessing.spatialFilter.enable else "off" + print(f"Spatial filter {state}") + if key == ord("t"): + StereoConfigHandler.newConfig = True + StereoConfigHandler.config.postProcessing.temporalFilter.enable = not StereoConfigHandler.config.postProcessing.temporalFilter.enable + state = "on" if StereoConfigHandler.config.postProcessing.temporalFilter.enable else "off" + print(f"Temporal filter {state}") + if key == ord("s"): + StereoConfigHandler.newConfig = True + StereoConfigHandler.config.postProcessing.speckleFilter.enable = not StereoConfigHandler.config.postProcessing.speckleFilter.enable + state = "on" if StereoConfigHandler.config.postProcessing.speckleFilter.enable else "off" + print(f"Speckle filter {state}") + if key == ord("r"): + StereoConfigHandler.newConfig = True + temporalSettings = [dai.StereoDepthConfig.PostProcessing.TemporalFilter.PersistencyMode.PERSISTENCY_OFF, + dai.StereoDepthConfig.PostProcessing.TemporalFilter.PersistencyMode.VALID_8_OUT_OF_8, + dai.StereoDepthConfig.PostProcessing.TemporalFilter.PersistencyMode.VALID_2_IN_LAST_3, + dai.StereoDepthConfig.PostProcessing.TemporalFilter.PersistencyMode.VALID_2_IN_LAST_4, + dai.StereoDepthConfig.PostProcessing.TemporalFilter.PersistencyMode.VALID_2_OUT_OF_8, + dai.StereoDepthConfig.PostProcessing.TemporalFilter.PersistencyMode.VALID_1_IN_LAST_2, + dai.StereoDepthConfig.PostProcessing.TemporalFilter.PersistencyMode.VALID_1_IN_LAST_5, + dai.StereoDepthConfig.PostProcessing.TemporalFilter.PersistencyMode.VALID_1_IN_LAST_8, + dai.StereoDepthConfig.PostProcessing.TemporalFilter.PersistencyMode.PERSISTENCY_INDEFINITELY, + ] + currentTemporal = StereoConfigHandler.config.postProcessing.temporalFilter.persistencyMode + nextTemporal = temporalSettings[(temporalSettings.index(currentTemporal)+1) % len(temporalSettings)] + print(f"Changing temporal persistency to {nextTemporal.name} from {currentTemporal.name}") + StereoConfigHandler.config.postProcessing.temporalFilter.persistencyMode = nextTemporal + if key == ord("n"): + StereoConfigHandler.newConfig = True + decimationSettings = [dai.StereoDepthConfig.PostProcessing.DecimationFilter.DecimationMode.PIXEL_SKIPPING, + dai.StereoDepthConfig.PostProcessing.DecimationFilter.DecimationMode.NON_ZERO_MEDIAN, + dai.StereoDepthConfig.PostProcessing.DecimationFilter.DecimationMode.NON_ZERO_MEAN, + ] + currentDecimation = StereoConfigHandler.config.postProcessing.decimationFilter.decimationMode + nextDecimation = decimationSettings[(decimationSettings.index(currentDecimation)+1) % len(decimationSettings)] + print(f"Changing decimation mode to {nextDecimation.name} from {currentDecimation.name}") + StereoConfigHandler.config.postProcessing.decimationFilter.decimationMode = nextDecimation + if key == ord("a"): + StereoConfigHandler.newConfig = True + alignmentSettings = [dai.StereoDepthConfig.AlgorithmControl.DepthAlign.RECTIFIED_RIGHT, + dai.StereoDepthConfig.AlgorithmControl.DepthAlign.RECTIFIED_LEFT, + dai.StereoDepthConfig.AlgorithmControl.DepthAlign.CENTER, + ] + currentAlignment = StereoConfigHandler.config.algorithmControl.depthAlign + nextAlignment = alignmentSettings[(alignmentSettings.index(currentAlignment)+1) % len(alignmentSettings)] + print(f"Changing alignment mode to {nextAlignment.name} from {currentAlignment.name}") + StereoConfigHandler.config.algorithmControl.depthAlign = nextAlignment + elif key == ord("c"): + StereoConfigHandler.newConfig = True + censusSettings = [dai.StereoDepthConfig.CensusTransform.KernelSize.AUTO, dai.StereoDepthConfig.CensusTransform.KernelSize.KERNEL_5x5, dai.StereoDepthConfig.CensusTransform.KernelSize.KERNEL_7x7, dai.StereoDepthConfig.CensusTransform.KernelSize.KERNEL_7x9] + currentCensus = StereoConfigHandler.config.censusTransform.kernelSize + nextCensus = censusSettings[(censusSettings.index(currentCensus)+1) % len(censusSettings)] + if nextCensus != dai.StereoDepthConfig.CensusTransform.KernelSize.AUTO: + censusGridSize = [(5,5), (7,7), (7,9)] + censusDefaultMask = [np.uint64(0XA82415), np.uint64(0XAA02A8154055), np.uint64(0X2AA00AA805540155)] + censusGrid = censusGridSize[nextCensus] + censusMask = censusDefaultMask[nextCensus] + StereoConfigHandler.censusMaskHandler = StereoConfigHandler.CensusMaskHandler("Census mask", censusGrid) + StereoConfigHandler.censusMaskHandler.setMask(censusMask) + else: + print("Census mask config is not available in AUTO census kernel mode. Change using the 'c' key") + StereoConfigHandler.config.censusTransform.kernelMask = 0 + StereoConfigHandler.censusMaskHandler.destroyWindow() + print(f"Changing census transform to {nextCensus.name} from {currentCensus.name}") + StereoConfigHandler.config.censusTransform.kernelSize = nextCensus + elif key == ord("d"): + StereoConfigHandler.newConfig = True + dispRangeSettings = [dai.StereoDepthConfig.CostMatching.DisparityWidth.DISPARITY_64, dai.StereoDepthConfig.CostMatching.DisparityWidth.DISPARITY_96] + currentDispRange = StereoConfigHandler.config.costMatching.disparityWidth + nextDispRange = dispRangeSettings[(dispRangeSettings.index(currentDispRange)+1) % len(dispRangeSettings)] + print(f"Changing disparity range to {nextDispRange.name} from {currentDispRange.name}") + StereoConfigHandler.config.costMatching.disparityWidth = nextDispRange + elif key == ord("f"): + StereoConfigHandler.newConfig = True + StereoConfigHandler.config.costMatching.enableCompanding = not StereoConfigHandler.config.costMatching.enableCompanding + state = "on" if StereoConfigHandler.config.costMatching.enableCompanding else "off" + print(f"Companding {state}") + elif key == ord("v"): + StereoConfigHandler.newConfig = True + StereoConfigHandler.config.censusTransform.enableMeanMode = not StereoConfigHandler.config.censusTransform.enableMeanMode + state = "on" if StereoConfigHandler.config.censusTransform.enableMeanMode else "off" + print(f"Census transform mean mode {state}") + elif key == ord("1"): + StereoConfigHandler.newConfig = True + StereoConfigHandler.config.algorithmControl.enableLeftRightCheck = not StereoConfigHandler.config.algorithmControl.enableLeftRightCheck + state = "on" if StereoConfigHandler.config.algorithmControl.enableLeftRightCheck else "off" + print(f"LR-check {state}") + elif key == ord("2"): + StereoConfigHandler.newConfig = True + StereoConfigHandler.config.algorithmControl.enableSubpixel = not StereoConfigHandler.config.algorithmControl.enableSubpixel + state = "on" if StereoConfigHandler.config.algorithmControl.enableSubpixel else "off" + print(f"Subpixel {state}") + elif key == ord("3"): + StereoConfigHandler.newConfig = True + StereoConfigHandler.config.algorithmControl.enableExtended = not StereoConfigHandler.config.algorithmControl.enableExtended + state = "on" if StereoConfigHandler.config.algorithmControl.enableExtended else "off" + print(f"Extended {state}") + + censusMaskChanged = False + if StereoConfigHandler.censusMaskHandler is not None: + censusMaskChanged = StereoConfigHandler.censusMaskHandler.isChanged() + if censusMaskChanged: + StereoConfigHandler.config.censusTransform.kernelMask = StereoConfigHandler.censusMaskHandler.getMask() + StereoConfigHandler.newConfig = True + + StereoConfigHandler.sendConfig(stereoDepthConfigInQueue) + + def sendConfig(stereoDepthConfigInQueue): + if StereoConfigHandler.newConfig: + StereoConfigHandler.newConfig = False + configMessage = dai.StereoDepthConfig() + configMessage.set(StereoConfigHandler.config) + stereoDepthConfigInQueue.send(configMessage) + + def updateDefaultConfig(config): + StereoConfigHandler.config = config + + def registerWindow(stream): + cv2.namedWindow(stream, cv2.WINDOW_NORMAL) + + StereoConfigHandler.trConfidence.append(StereoConfigHandler.Trackbar("Disparity confidence", stream, 0, 255, StereoConfigHandler.config.costMatching.confidenceThreshold, StereoConfigHandler.trackbarConfidence)) + StereoConfigHandler.trSigma.append(StereoConfigHandler.Trackbar("Bilateral sigma", stream, 0, 100, StereoConfigHandler.config.postProcessing.bilateralSigmaValue, StereoConfigHandler.trackbarSigma)) + StereoConfigHandler.trLrCheck.append(StereoConfigHandler.Trackbar("LR-check threshold", stream, 0, 16, StereoConfigHandler.config.algorithmControl.leftRightCheckThreshold, StereoConfigHandler.trackbarLrCheckThreshold)) + StereoConfigHandler.trFractionalBits.append(StereoConfigHandler.Trackbar("Subpixel fractional bits", stream, 3, 5, StereoConfigHandler.config.algorithmControl.subpixelFractionalBits, StereoConfigHandler.trackbarFractionalBits)) + StereoConfigHandler.trDisparityShift.append(StereoConfigHandler.Trackbar("Disparity shift", stream, 0, 100, StereoConfigHandler.config.algorithmControl.disparityShift, StereoConfigHandler.trackbarDisparityShift)) + StereoConfigHandler.trCenterAlignmentShift.append(StereoConfigHandler.Trackbar("Center alignment shift factor", stream, 0, 100, StereoConfigHandler.config.algorithmControl.centerAlignmentShiftFactor, StereoConfigHandler.trackbarCenterAlignmentShift)) + StereoConfigHandler.trInvalidateEdgePixels.append(StereoConfigHandler.Trackbar("Invalidate edge pixels", stream, 0, 100, StereoConfigHandler.config.algorithmControl.numInvalidateEdgePixels, StereoConfigHandler.trackbarInvalidateEdgePixels)) + StereoConfigHandler.trLineqAlpha.append(StereoConfigHandler.Trackbar("Linear equation alpha", stream, 0, 15, StereoConfigHandler.config.costMatching.linearEquationParameters.alpha, StereoConfigHandler.trackbarLineqAlpha)) + StereoConfigHandler.trLineqBeta.append(StereoConfigHandler.Trackbar("Linear equation beta", stream, 0, 15, StereoConfigHandler.config.costMatching.linearEquationParameters.beta, StereoConfigHandler.trackbarLineqBeta)) + StereoConfigHandler.trLineqThreshold.append(StereoConfigHandler.Trackbar("Linear equation threshold", stream, 0, 255, StereoConfigHandler.config.costMatching.linearEquationParameters.threshold, StereoConfigHandler.trackbarLineqThreshold)) + StereoConfigHandler.trCostAggregationP1.append(StereoConfigHandler.Trackbar("Cost aggregation P1", stream, 0, 500, StereoConfigHandler.config.costAggregation.horizontalPenaltyCostP1, StereoConfigHandler.trackbarCostAggregationP1)) + StereoConfigHandler.trCostAggregationP2.append(StereoConfigHandler.Trackbar("Cost aggregation P2", stream, 0, 500, StereoConfigHandler.config.costAggregation.horizontalPenaltyCostP2, StereoConfigHandler.trackbarCostAggregationP2)) + StereoConfigHandler.trTemporalAlpha.append(StereoConfigHandler.Trackbar("Temporal filter alpha", stream, 0, 100, int(StereoConfigHandler.config.postProcessing.temporalFilter.alpha*100), StereoConfigHandler.trackbarTemporalFilterAlpha)) + StereoConfigHandler.trTemporalDelta.append(StereoConfigHandler.Trackbar("Temporal filter delta", stream, 0, 100, StereoConfigHandler.config.postProcessing.temporalFilter.delta, StereoConfigHandler.trackbarTemporalFilterDelta)) + StereoConfigHandler.trSpatialAlpha.append(StereoConfigHandler.Trackbar("Spatial filter alpha", stream, 0, 100, int(StereoConfigHandler.config.postProcessing.spatialFilter.alpha*100), StereoConfigHandler.trackbarSpatialFilterAlpha)) + StereoConfigHandler.trSpatialDelta.append(StereoConfigHandler.Trackbar("Spatial filter delta", stream, 0, 100, StereoConfigHandler.config.postProcessing.spatialFilter.delta, StereoConfigHandler.trackbarSpatialFilterDelta)) + StereoConfigHandler.trSpatialHoleFilling.append(StereoConfigHandler.Trackbar("Spatial filter hole filling radius", stream, 0, 16, StereoConfigHandler.config.postProcessing.spatialFilter.holeFillingRadius, StereoConfigHandler.trackbarSpatialFilterHoleFillingRadius)) + StereoConfigHandler.trSpatialNumIterations.append(StereoConfigHandler.Trackbar("Spatial filter number of iterations", stream, 0, 4, StereoConfigHandler.config.postProcessing.spatialFilter.numIterations, StereoConfigHandler.trackbarSpatialFilterNumIterations)) + StereoConfigHandler.trThresholdMinRange.append(StereoConfigHandler.Trackbar("Threshold filter min range", stream, 0, 65, StereoConfigHandler.config.postProcessing.thresholdFilter.minRange, StereoConfigHandler.trackbarThresholdMinRange)) + StereoConfigHandler.trThresholdMaxRange.append(StereoConfigHandler.Trackbar("Threshold filter max range", stream, 0, 65, StereoConfigHandler.config.postProcessing.thresholdFilter.maxRange, StereoConfigHandler.trackbarThresholdMaxRange)) + StereoConfigHandler.trSpeckleRange.append(StereoConfigHandler.Trackbar("Speckle filter range", stream, 0, 240, StereoConfigHandler.config.postProcessing.speckleFilter.speckleRange, StereoConfigHandler.trackbarSpeckleRange)) + StereoConfigHandler.trDecimationFactor.append(StereoConfigHandler.Trackbar("Decimation factor", stream, 1, 4, StereoConfigHandler.config.postProcessing.decimationFilter.decimationFactor, StereoConfigHandler.trackbarDecimationFactor)) + + def __init__(self, config): + print("Control median filter using the 'm' key.") + print("Control census transform kernel size using the 'c' key.") + print("Control disparity search range using the 'd' key.") + print("Control disparity companding using the 'f' key.") + print("Control census transform mean mode using the 'v' key.") + print("Control depth alignment using the 'a' key.") + print("Control decimation algorithm using the 'a' key.") + print("Control temporal persistency mode using the 'r' key.") + print("Control spatial filter using the 'w' key.") + print("Control temporal filter using the 't' key.") + print("Control speckle filter using the 's' key.") + print("Control left-right check mode using the '1' key.") + print("Control subpixel mode using the '2' key.") + print("Control extended mode using the '3' key.") + + StereoConfigHandler.config = config + + if StereoConfigHandler.config.censusTransform.kernelSize != dai.StereoDepthConfig.CensusTransform.KernelSize.AUTO: + censusMask = StereoConfigHandler.config.censusTransform.kernelMask + censusGridSize = [(5,5), (7,7), (7,9)] + censusGrid = censusGridSize[StereoConfigHandler.config.censusTransform.kernelSize] + if StereoConfigHandler.config.censusTransform.kernelMask == 0: + censusDefaultMask = [np.uint64(0xA82415), np.uint64(0xAA02A8154055), np.uint64(0x2AA00AA805540155)] + censusMask = censusDefaultMask[StereoConfigHandler.config.censusTransform.kernelSize] + StereoConfigHandler.censusMaskHandler = StereoConfigHandler.CensusMaskHandler("Census mask", censusGrid) + StereoConfigHandler.censusMaskHandler.setMask(censusMask) + else: + print("Census mask config is not available in AUTO Census kernel mode. Change using the 'c' key") + + +# StereoDepth initial config options. +outDepth = True # Disparity by default +outConfidenceMap = True # Output disparity confidence map +outRectified = True # Output and display rectified streams +lrcheck = True # Better handling for occlusions +extended = False # Closer-in minimum depth, disparity range is doubled. Unsupported for now. +subpixel = True # Better accuracy for longer distance, fractional disparity 32-levels + +width = 1280 +height = 800 + +xoutStereoCfg = None + +# Create pipeline +pipeline = dai.Pipeline() + +# Define sources and outputs +stereo = pipeline.create(dai.node.StereoDepth) + +monoLeft = pipeline.create(dai.node.XLinkIn) +monoRight = pipeline.create(dai.node.XLinkIn) +xinStereoDepthConfig = pipeline.create(dai.node.XLinkIn) + +xoutLeft = pipeline.create(dai.node.XLinkOut) +xoutRight = pipeline.create(dai.node.XLinkOut) +xoutDepth = pipeline.create(dai.node.XLinkOut) +xoutConfMap = pipeline.create(dai.node.XLinkOut) +xoutDisparity = pipeline.create(dai.node.XLinkOut) +xoutRectifLeft = pipeline.create(dai.node.XLinkOut) +xoutRectifRight = pipeline.create(dai.node.XLinkOut) +xoutStereoCfg = pipeline.create(dai.node.XLinkOut) +if args.debug: + xoutDebugLrCheckIt1 = pipeline.create(dai.node.XLinkOut) + xoutDebugLrCheckIt2 = pipeline.create(dai.node.XLinkOut) + xoutDebugExtLrCheckIt1 = pipeline.create(dai.node.XLinkOut) + xoutDebugExtLrCheckIt2 = pipeline.create(dai.node.XLinkOut) +if args.dumpdisparitycostvalues: + xoutDebugCostDump = pipeline.create(dai.node.XLinkOut) + +xinStereoDepthConfig.setStreamName("stereoDepthConfig") +monoLeft.setStreamName("in_left") +monoRight.setStreamName("in_right") + +xoutLeft.setStreamName("left") +xoutRight.setStreamName("right") +xoutDepth.setStreamName("depth") +xoutConfMap.setStreamName("confidence_map") +xoutDisparity.setStreamName("disparity") +xoutRectifLeft.setStreamName("rectified_left") +xoutRectifRight.setStreamName("rectified_right") +xoutStereoCfg.setStreamName("stereo_cfg") +if args.debug: + xoutDebugLrCheckIt1.setStreamName("disparity_lr_check_iteration1") + xoutDebugLrCheckIt2.setStreamName("disparity_lr_check_iteration2") + xoutDebugExtLrCheckIt1.setStreamName("disparity_ext_lr_check_iteration1") + xoutDebugExtLrCheckIt2.setStreamName("disparity_ext_lr_check_iteration2") +if args.dumpdisparitycostvalues: + xoutDebugCostDump.setStreamName("disparity_cost_dump") + +# Properties +stereo.setDefaultProfilePreset(dai.node.StereoDepth.PresetMode.HIGH_DENSITY) +stereo.setRectifyEdgeFillColor(0) # Black, to better see the cutout +stereo.setLeftRightCheck(lrcheck) +stereo.setExtendedDisparity(extended) +stereo.setSubpixel(subpixel) + +# Switching depthAlign mode at runtime is not supported while aligning to a specific camera is enabled +# stereo.setDepthAlign(dai.CameraBoardSocket.LEFT) + +# allocates resources for worst case scenario +# allowing runtime switch of stereo modes +stereo.setRuntimeModeSwitch(True) + +# Linking +monoLeft.out.link(stereo.left) +monoRight.out.link(stereo.right) +xinStereoDepthConfig.out.link(stereo.inputConfig) +stereo.syncedLeft.link(xoutLeft.input) +stereo.syncedRight.link(xoutRight.input) +if outDepth: + stereo.depth.link(xoutDepth.input) +if outConfidenceMap: + stereo.confidenceMap.link(xoutConfMap.input) +stereo.disparity.link(xoutDisparity.input) +if outRectified: + stereo.rectifiedLeft.link(xoutRectifLeft.input) + stereo.rectifiedRight.link(xoutRectifRight.input) +stereo.outConfig.link(xoutStereoCfg.input) +if args.debug: + stereo.debugDispLrCheckIt1.link(xoutDebugLrCheckIt1.input) + stereo.debugDispLrCheckIt2.link(xoutDebugLrCheckIt2.input) + stereo.debugExtDispLrCheckIt1.link(xoutDebugExtLrCheckIt1.input) + stereo.debugExtDispLrCheckIt2.link(xoutDebugExtLrCheckIt2.input) +if args.dumpdisparitycostvalues: + stereo.debugDispCostDump.link(xoutDebugCostDump.input) + + +StereoConfigHandler(stereo.initialConfig.get()) +StereoConfigHandler.registerWindow("Stereo control panel") + +# stereo.setPostProcessingHardwareResources(3, 3) + +stereo.setInputResolution(width, height) +stereo.setRectification(False) +baseline = 75 +fov = 71.86 +focal = width / (2 * math.tan(fov / 2 / 180 * math.pi)) + +streams = ["left", "right"] +if outRectified: + streams.extend(["rectified_left", "rectified_right"]) +streams.append("disparity") +if outDepth: + streams.append("depth") +if outConfidenceMap: + streams.append("confidence_map") +debugStreams = [] +if args.debug: + debugStreams.extend(["disparity_lr_check_iteration1", "disparity_lr_check_iteration2"]) + debugStreams.extend(["disparity_ext_lr_check_iteration1", "disparity_ext_lr_check_iteration2"]) +if args.dumpdisparitycostvalues: + debugStreams.append("disparity_cost_dump") + +def convertToCv2Frame(name, image, config): + + maxDisp = config.getMaxDisparity() + subpixelLevels = pow(2, config.get().algorithmControl.subpixelFractionalBits) + subpixel = config.get().algorithmControl.enableSubpixel + dispIntegerLevels = maxDisp if not subpixel else maxDisp / subpixelLevels + + frame = image.getFrame() + + # frame.tofile(name+".raw") + + if name == "depth": + dispScaleFactor = baseline * focal + with np.errstate(divide="ignore"): + frame = dispScaleFactor / frame + + frame = (frame * 255. / dispIntegerLevels).astype(np.uint8) + frame = cv2.applyColorMap(frame, cv2.COLORMAP_HOT) + elif "confidence_map" in name: + pass + elif name == "disparity_cost_dump": + # frame.tofile(name+".raw") + pass + elif "disparity" in name: + if 1: # Optionally, extend disparity range to better visualize it + frame = (frame * 255. / maxDisp).astype(np.uint8) + return frame + # if 1: # Optionally, apply a color map + # frame = cv2.applyColorMap(frame, cv2.COLORMAP_HOT) + + return frame + +class DatasetManager: + def __init__(self, path): + self.path = path + self.index = 0 + self.names = [d for d in os.listdir(path) if os.path.isdir(os.path.join(path, d))] + if len(self.names) == 0: + raise RuntimeError("No dataset found at {}".format(path)) + + def get(self): + return os.path.join(self.path, self.names[self.index]) + + def get_name(self): + return self.names[self.index] + + def next(self): + self.index = (self.index + 1) % len(self.names) + return self.get() + + def prev(self): + self.index = (self.index - 1) % len(self.names) + return self.get() + + +def calculate_err_measures(gt_img, oak_img): + assert gt_img.shape == oak_img.shape + + gt_mask = gt_img != np.inf + oak_mask = oak_img != np.inf + mask = gt_mask & oak_mask + + gt_img[~gt_mask] = 0. + oak_img[~mask] = 0. + err = np.abs(gt_img - oak_img) + + n = np.sum(gt_mask) + invalid = np.sum(gt_mask & ~oak_mask) + + bad05 = np.sum(mask & (err > 0.5)) + bad1 = np.sum(mask & (err > 1.)) + bad2 = np.sum(mask & (err > 2.)) + bad4 = np.sum(mask & (err > 4.)) + sum_err = np.sum(err[mask]) + sum_sq_err = np.sum(err[mask] ** 2) + errs = err[mask] + + bad05_p = 100. * bad05 / n + total_bad05_p = 100. * (bad05 + invalid) / n + bad1_p = 100. * bad1 / n + total_bad1_p = 100. * (bad1 + invalid) / n + bad2_p = 100. * bad2 / n + total_bad2_p = 100. * (bad2 + invalid) / n + bad4_p = 100. * bad4 / n + total_bad4_p = 100. * (bad4 + invalid) / n + invalid_p = 100. * invalid / n + avg_err = sum_err / (n - invalid) + mse = sum_sq_err / (n - invalid) + a50 = np.percentile(errs, 50) + a90 = np.percentile(errs, 90) + a95 = np.percentile(errs, 95) + a99 = np.percentile(errs, 99) + + return { + "bad0.5": bad05_p, + "total_bad0.5": total_bad05_p, + "bad1": bad1_p, + "total_bad1": total_bad1_p, + "bad2": bad2_p, + "total_bad2": total_bad2_p, + "bad4": bad4_p, + "total_bad4": total_bad4_p, + "invalid": invalid_p, + "avg_err": avg_err, + "mse": mse, + "a50": a50, + "a90": a90, + "a95": a95, + "a99": a99 + } + +def show_evaluation(img_name, evals): + cv2.namedWindow("Evaluation", cv2.WINDOW_NORMAL) + font = cv2.FONT_HERSHEY_SIMPLEX + font_scale = 2 + thickness = 3 + color = (0, 0, 0) + lines = [ + f"Name: {img_name}", + f"Bad0.5: {evals['bad0.5']:.2f}%", + f"Total Bad0.5: {evals['total_bad0.5']:.2f}%", + f"Bad1: {evals['bad1']:.2f}%", + f"Total Bad1: {evals['total_bad1']:.2f}%", + f"Bad2: {evals['bad2']:.2f}%", + f"Total Bad2: {evals['total_bad2']:.2f}%", + f"Bad4: {evals['bad4']:.2f}%", + f"Total Bad4: {evals['total_bad4']:.2f}%", + f"Invalid: {evals['invalid']:.2f}%", + f"Avg Err: {evals['avg_err']:.2f}", + f"MSE: {evals['mse']:.2f}", + f"A50: {evals['a50']:.2f}", + f"A90: {evals['a90']:.2f}", + f"A95: {evals['a95']:.2f}", + f"A99: {evals['a99']:.2f}" + ] + sizes = [cv2.getTextSize(line, font, font_scale, thickness) for line in lines] + sizes = [(size[0][0], size[0][1] + size[1], size[1]) for size in sizes] + max_width = max([size[0] for size in sizes]) + total_height = sum([size[1] for size in sizes]) + (len(lines) - 1) * thickness + img = np.ones((total_height + thickness, max_width, 3), dtype=np.uint8) * 255 + y = 0 + for line, size in zip(lines, sizes): + cv2.putText(img, line, (0, y + size[1] - size[2]), font, font_scale, color, thickness) + y += size[1] + thickness + cv2.imshow("Evaluation", img) + +def show_debug_disparity(gt_img, oak_img): + def rescale_img(img): + img[img == np.inf] = 0. + img = cv2.resize(img, (1280, 800), interpolation=cv2.INTER_AREA) + return img.astype(np.uint16) + + gt_img = rescale_img(gt_img) + oak_img = rescale_img(oak_img) + maxv = max(gt_img.max(), oak_img.max()) + gt_img = (gt_img * 255. / maxv).astype(np.uint8) + oak_img = (oak_img * 255. / maxv).astype(np.uint8) + cv2.imshow("GT", gt_img) + cv2.imshow("OAK", oak_img) + +dataset = DatasetManager(args.dataset) + +print("Connecting and starting the pipeline") +# Connect to device and start pipeline +with dai.Device(pipeline) as device: + + stereoDepthConfigInQueue = device.getInputQueue("stereoDepthConfig") + inStreams = ["in_left", "in_right"] + inStreamsCameraID = [dai.CameraBoardSocket.CAM_B, dai.CameraBoardSocket.CAM_C] + in_q_list = [] + for s in inStreams: + q = device.getInputQueue(s) + in_q_list.append(q) + + # Create a receive queue for each stream + q_list = [] + for s in streams: + q = device.getOutputQueue(s, 8, blocking=False) + q_list.append(q) + + inCfg = device.getOutputQueue("stereo_cfg", 8, blocking=False) + + # Need to set a timestamp for input frames, for the sync stage in Stereo node + timestamp_ms = 0 + prevQueues = q_list.copy() + while True: + # Handle input streams, if any + if in_q_list: + dataset_size = 1 # Number of image pairs + frame_interval_ms = 50 + for i, q in enumerate(in_q_list): + path = os.path.join(dataset.get(), f"im{i}.png") + data = cv2.imread(path, cv2.IMREAD_GRAYSCALE) + data = cv2.resize(data, (width, height), interpolation = cv2.INTER_AREA) + data = data.reshape(height*width) + tstamp = datetime.timedelta(seconds = timestamp_ms // 1000, + milliseconds = timestamp_ms % 1000) + img = dai.ImgFrame() + img.setData(data) + img.setTimestamp(tstamp) + img.setInstanceNum(inStreamsCameraID[i]) + img.setType(dai.ImgFrame.Type.RAW8) + img.setWidth(width) + img.setHeight(height) + q.send(img) + # print("Sent frame: {:25s}".format(path), "timestamp_ms:", timestamp_ms) + timestamp_ms += frame_interval_ms + sleep(frame_interval_ms / 1000) + + # Load GT disparity + gt_disparity = None + if args.evaluate: + gt_disparity = read_pfm(os.path.join(dataset.get(), f"disp0.pfm"))[0] + + # Handle output streams + currentConfig = inCfg.get() + + lrCheckEnabled = currentConfig.get().algorithmControl.enableLeftRightCheck + extendedEnabled = currentConfig.get().algorithmControl.enableExtended + queues = q_list.copy() + + if args.dumpdisparitycostvalues: + q = device.getOutputQueue("disparity_cost_dump", 8, blocking=False) + queues.append(q) + + if args.debug: + q_list_debug = [] + + activeDebugStreams = [] + if lrCheckEnabled: + activeDebugStreams.extend(["disparity_lr_check_iteration1", "disparity_lr_check_iteration2"]) + if extendedEnabled: + activeDebugStreams.extend(["disparity_ext_lr_check_iteration1"]) + if lrCheckEnabled: + activeDebugStreams.extend(["disparity_ext_lr_check_iteration2"]) + + for s in activeDebugStreams: + q = device.getOutputQueue(s, 8, blocking=False) + q_list_debug.append(q) + + queues.extend(q_list_debug) + + def ListDiff(li1, li2): + return list(set(li1) - set(li2)) + list(set(li2) - set(li1)) + + diff = ListDiff(prevQueues, queues) + for s in diff: + name = s.getName() + cv2.destroyWindow(name) + prevQueues = queues.copy() + + disparity = None + for q in queues: + if q.getName() in ["left", "right"]: continue + data = q.get() + if q.getName() == "disparity": + disparity = data.getFrame() + frame = convertToCv2Frame(q.getName(), data, currentConfig) + cv2.imshow(q.getName(), frame) + + if disparity is not None and gt_disparity is not None: + subpixel_bits = 1 << currentConfig.get().algorithmControl.subpixelFractionalBits + subpixel_enabled = currentConfig.get().algorithmControl.enableSubpixel + width_scale = float(gt_disparity.shape[1]) / float(disparity.shape[1]) + + disparity = disparity.astype(np.float32) + if subpixel_enabled: + disparity = disparity / subpixel_bits + disparity = disparity * width_scale + disparity = cv2.resize(disparity, (gt_disparity.shape[1], gt_disparity.shape[0]), interpolation = cv2.INTER_LINEAR) + disparity[disparity == 0.] = np.inf + + # show_debug_disparity(gt_disparity, disparity) + err_vals = calculate_err_measures(gt_disparity, disparity) + show_evaluation(dataset.get_name(), err_vals) + + key = cv2.waitKey(1) + if key == ord("q"): + break + elif key == ord("["): + dataset.next() + elif key == ord("]"): + dataset.prev() + + StereoConfigHandler.handleKeypress(key, stereoDepthConfigInQueue) From 9bc250b2d6e03bff25db24b6b658ce7a5ce9f6de Mon Sep 17 00:00:00 2001 From: asahtik Date: Tue, 8 Aug 2023 09:57:33 +0200 Subject: [PATCH 41/87] Merged scripts, improved dataset downloading, arguments --- examples/StereoDepth/read_write_pfm.py | 65 -- .../StereoDepth/stereo_depth_evaluation.py | 918 ------------------ .../StereoDepth/stereo_depth_from_host.py | 411 ++++++-- 3 files changed, 326 insertions(+), 1068 deletions(-) delete mode 100644 examples/StereoDepth/read_write_pfm.py delete mode 100644 examples/StereoDepth/stereo_depth_evaluation.py diff --git a/examples/StereoDepth/read_write_pfm.py b/examples/StereoDepth/read_write_pfm.py deleted file mode 100644 index 03fc02927..000000000 --- a/examples/StereoDepth/read_write_pfm.py +++ /dev/null @@ -1,65 +0,0 @@ -import numpy as np -import re -import sys - -def read_pfm(file): - file = open(file, "rb") - - color = None - width = None - height = None - scale = None - endian = None - - header = file.readline().rstrip() - if header.decode("ascii") == "PF": - color = True - elif header.decode("ascii") == "Pf": - color = False - else: - raise Exception("Not a PFM file.") - - dim_match = re.search(r"(\d+)\s(\d+)", file.readline().decode("ascii")) - if dim_match: - width, height = map(int, dim_match.groups()) - else: - raise Exception("Malformed PFM header.") - - scale = float(file.readline().rstrip()) - if scale < 0: # little-endian - endian = "<" - scale = -scale - else: - endian = ">" # big-endian - - data = np.fromfile(file, endian + "f") - shape = (height, width, 3) if color else (height, width) - return np.flip(np.reshape(data, shape), axis=0), scale - -def write_pfm(file, image, scale = 1. / 255): - image = np.flip(image, axis=0) - file = open(file, "wb") - - color = None - - if image.dtype.name != "float32": - raise Exception("Image dtype must be float32.") - - if len(image.shape) == 3 and image.shape[2] == 3: # color image - color = True - elif len(image.shape) == 2 or len(image.shape) == 3 and image.shape[2] == 1: # greyscale - color = False - else: - raise Exception("Image must have H x W x 3, H x W x 1 or H x W dimensions.") - - file.write(b"PF\n" if color else b"Pf\n") - file.write(b"%d %d\n" % (image.shape[1], image.shape[0])) - - endian = image.dtype.byteorder - - if endian == "<" or endian == "=" and sys.byteorder == "little": - scale = -scale - - file.write(b"%f\n" % scale) - - image.tofile(file) diff --git a/examples/StereoDepth/stereo_depth_evaluation.py b/examples/StereoDepth/stereo_depth_evaluation.py deleted file mode 100644 index bd154a760..000000000 --- a/examples/StereoDepth/stereo_depth_evaluation.py +++ /dev/null @@ -1,918 +0,0 @@ -#!/usr/bin/env python3 - -import cv2 -import numpy as np -import depthai as dai -from time import sleep -import datetime -import argparse -from pathlib import Path -import math -import os -from read_write_pfm import read_pfm - -datasetDefault = str((Path(__file__).parent / Path("models/dataset")).resolve().absolute()) -parser = argparse.ArgumentParser() -parser.add_argument("-p", "--dataset", nargs="?", help="Path to dataset", default=datasetDefault) -parser.add_argument("-d", "--debug", action="store_true", help="Enable debug outputs.") -parser.add_argument("-e", "--evaluate", action="store_true", help="Output GT evaluation.") -parser.add_argument("-dumpdispcost", "--dumpdisparitycostvalues", action="store_true", help="Dumps the disparity cost values for each disparity range. 96 byte for each pixel.") -parser.add_argument("--download", action="store_true", help="Downloads the 2014 Middlebury dataset.", default=False) -args = parser.parse_args() - -if not Path(args.dataset).exists(): - if not args.download: - import sys - raise FileNotFoundError(f"Dataset path does not exist, use the --dataset argument to specify the path.") - else: - os.makedirs(args.dataset) - -def download_2014_middlebury(data_path): - import requests, zipfile, io, re - url = "https://vision.middlebury.edu/stereo/data/scenes2014/zip/" - r = requests.get(url) - c = r.content - reg = re.compile(r"href=('|\")(.+\.zip)('|\")") - matches = reg.findall(c.decode("utf-8")) - files = [m[1] for m in matches] - - for f in files: - print(f"Downloading {f} from {url + f}") - r = requests.get(url + f) - print(f"Extracting {f}") - z = zipfile.ZipFile(io.BytesIO(r.content)) - z.extractall(os.path.join(data_path, f[:-4])) - -if args.download: - download_2014_middlebury(args.dataset) - -class StereoConfigHandler: - - class Trackbar: - def __init__(self, trackbarName, windowName, minValue, maxValue, defaultValue, handler): - self.min = minValue - self.max = maxValue - self.windowName = windowName - self.trackbarName = trackbarName - cv2.createTrackbar(trackbarName, windowName, minValue, maxValue, handler) - cv2.setTrackbarPos(trackbarName, windowName, defaultValue) - - def set(self, value): - if value < self.min: - value = self.min - print(f"{self.trackbarName} min value is {self.min}") - if value > self.max: - value = self.max - print(f"{self.trackbarName} max value is {self.max}") - cv2.setTrackbarPos(self.trackbarName, self.windowName, value) - - class CensusMaskHandler: - - stateColor = [(0, 0, 0), (255, 255, 255)] - gridHeight = 50 - gridWidth = 50 - - def fillRectangle(self, row, col): - src = self.gridList[row][col]["topLeft"] - dst = self.gridList[row][col]["bottomRight"] - - stateColor = self.stateColor[1] if self.gridList[row][col]["state"] else self.stateColor[0] - self.changed = True - - cv2.rectangle(self.gridImage, src, dst, stateColor, -1) - cv2.imshow(self.windowName, self.gridImage) - - - def clickCallback(self, event, x, y, flags, param): - if event == cv2.EVENT_LBUTTONDOWN: - col = x * (self.gridSize[1]) // self.width - row = y * (self.gridSize[0]) // self.height - self.gridList[row][col]["state"] = not self.gridList[row][col]["state"] - self.fillRectangle(row, col) - - - def __init__(self, windowName, gridSize): - self.gridSize = gridSize - self.windowName = windowName - self.changed = False - - cv2.namedWindow(self.windowName) - - self.width = StereoConfigHandler.CensusMaskHandler.gridWidth * self.gridSize[1] - self.height = StereoConfigHandler.CensusMaskHandler.gridHeight * self.gridSize[0] - - self.gridImage = np.zeros((self.height + 50, self.width, 3), np.uint8) - - cv2.putText(self.gridImage, "Click on grid to change mask!", (0, self.height+20), cv2.FONT_HERSHEY_SIMPLEX, 0.4, (255, 255, 255)) - cv2.putText(self.gridImage, "White: ON | Black: OFF", (0, self.height+40), cv2.FONT_HERSHEY_SIMPLEX, 0.4, (255, 255, 255)) - - self.gridList = [[dict() for _ in range(self.gridSize[1])] for _ in range(self.gridSize[0])] - - for row in range(self.gridSize[0]): - rowFactor = self.height // self.gridSize[0] - srcY = row*rowFactor + 1 - dstY = (row+1)*rowFactor - 1 - for col in range(self.gridSize[1]): - colFactor = self.width // self.gridSize[1] - srcX = col*colFactor + 1 - dstX = (col+1)*colFactor - 1 - src = (srcX, srcY) - dst = (dstX, dstY) - self.gridList[row][col]["topLeft"] = src - self.gridList[row][col]["bottomRight"] = dst - self.gridList[row][col]["state"] = False - self.fillRectangle(row, col) - - cv2.setMouseCallback(self.windowName, self.clickCallback) - cv2.imshow(self.windowName, self.gridImage) - - def getMask(self) -> np.uint64: - mask = np.uint64(0) - for row in range(self.gridSize[0]): - for col in range(self.gridSize[1]): - if self.gridList[row][col]["state"]: - pos = row*self.gridSize[1] + col - mask = np.bitwise_or(mask, np.uint64(1) << np.uint64(pos)) - - return mask - - def setMask(self, _mask: np.uint64): - mask = np.uint64(_mask) - for row in range(self.gridSize[0]): - for col in range(self.gridSize[1]): - pos = row*self.gridSize[1] + col - if np.bitwise_and(mask, np.uint64(1) << np.uint64(pos)): - self.gridList[row][col]["state"] = True - else: - self.gridList[row][col]["state"] = False - - self.fillRectangle(row, col) - - def isChanged(self): - changed = self.changed - self.changed = False - return changed - - def destroyWindow(self): - cv2.destroyWindow(self.windowName) - - - censusMaskHandler = None - newConfig = False - config = None - trSigma = list() - trConfidence = list() - trLrCheck = list() - trFractionalBits = list() - trLineqAlpha = list() - trLineqBeta = list() - trLineqThreshold = list() - trCostAggregationP1 = list() - trCostAggregationP2 = list() - trTemporalAlpha = list() - trTemporalDelta = list() - trThresholdMinRange = list() - trThresholdMaxRange = list() - trSpeckleRange = list() - trSpatialAlpha = list() - trSpatialDelta = list() - trSpatialHoleFilling = list() - trSpatialNumIterations = list() - trDecimationFactor = list() - trDisparityShift = list() - trCenterAlignmentShift = list() - trInvalidateEdgePixels = list() - - def trackbarSigma(value): - StereoConfigHandler.config.postProcessing.bilateralSigmaValue = value - StereoConfigHandler.newConfig = True - for tr in StereoConfigHandler.trSigma: - tr.set(value) - - def trackbarConfidence(value): - StereoConfigHandler.config.costMatching.confidenceThreshold = value - StereoConfigHandler.newConfig = True - for tr in StereoConfigHandler.trConfidence: - tr.set(value) - - def trackbarLrCheckThreshold(value): - StereoConfigHandler.config.algorithmControl.leftRightCheckThreshold = value - StereoConfigHandler.newConfig = True - for tr in StereoConfigHandler.trLrCheck: - tr.set(value) - - def trackbarFractionalBits(value): - StereoConfigHandler.config.algorithmControl.subpixelFractionalBits = value - StereoConfigHandler.newConfig = True - for tr in StereoConfigHandler.trFractionalBits: - tr.set(value) - - def trackbarLineqAlpha(value): - StereoConfigHandler.config.costMatching.linearEquationParameters.alpha = value - StereoConfigHandler.newConfig = True - for tr in StereoConfigHandler.trLineqAlpha: - tr.set(value) - - def trackbarLineqBeta(value): - StereoConfigHandler.config.costMatching.linearEquationParameters.beta = value - StereoConfigHandler.newConfig = True - for tr in StereoConfigHandler.trLineqBeta: - tr.set(value) - - def trackbarLineqThreshold(value): - StereoConfigHandler.config.costMatching.linearEquationParameters.threshold = value - StereoConfigHandler.newConfig = True - for tr in StereoConfigHandler.trLineqThreshold: - tr.set(value) - - def trackbarCostAggregationP1(value): - StereoConfigHandler.config.costAggregation.horizontalPenaltyCostP1 = value - StereoConfigHandler.config.costAggregation.verticalPenaltyCostP1 = value - StereoConfigHandler.newConfig = True - for tr in StereoConfigHandler.trCostAggregationP1: - tr.set(value) - - def trackbarCostAggregationP2(value): - StereoConfigHandler.config.costAggregation.horizontalPenaltyCostP2 = value - StereoConfigHandler.config.costAggregation.verticalPenaltyCostP2 = value - StereoConfigHandler.newConfig = True - for tr in StereoConfigHandler.trCostAggregationP2: - tr.set(value) - - def trackbarTemporalFilterAlpha(value): - StereoConfigHandler.config.postProcessing.temporalFilter.alpha = value / 100. - StereoConfigHandler.newConfig = True - for tr in StereoConfigHandler.trTemporalAlpha: - tr.set(value) - - def trackbarTemporalFilterDelta(value): - StereoConfigHandler.config.postProcessing.temporalFilter.delta = value - StereoConfigHandler.newConfig = True - for tr in StereoConfigHandler.trTemporalDelta: - tr.set(value) - - def trackbarSpatialFilterAlpha(value): - StereoConfigHandler.config.postProcessing.spatialFilter.alpha = value / 100. - StereoConfigHandler.newConfig = True - for tr in StereoConfigHandler.trSpatialAlpha: - tr.set(value) - - def trackbarSpatialFilterDelta(value): - StereoConfigHandler.config.postProcessing.spatialFilter.delta = value - StereoConfigHandler.newConfig = True - for tr in StereoConfigHandler.trSpatialDelta: - tr.set(value) - - def trackbarSpatialFilterHoleFillingRadius(value): - StereoConfigHandler.config.postProcessing.spatialFilter.holeFillingRadius = value - StereoConfigHandler.newConfig = True - for tr in StereoConfigHandler.trSpatialHoleFilling: - tr.set(value) - - def trackbarSpatialFilterNumIterations(value): - StereoConfigHandler.config.postProcessing.spatialFilter.numIterations = value - StereoConfigHandler.newConfig = True - for tr in StereoConfigHandler.trSpatialNumIterations: - tr.set(value) - - def trackbarThresholdMinRange(value): - StereoConfigHandler.config.postProcessing.thresholdFilter.minRange = value * 1000 - StereoConfigHandler.newConfig = True - for tr in StereoConfigHandler.trThresholdMinRange: - tr.set(value) - - def trackbarThresholdMaxRange(value): - StereoConfigHandler.config.postProcessing.thresholdFilter.maxRange = value * 1000 - StereoConfigHandler.newConfig = True - for tr in StereoConfigHandler.trThresholdMaxRange: - tr.set(value) - - def trackbarSpeckleRange(value): - StereoConfigHandler.config.postProcessing.speckleFilter.speckleRange = value - StereoConfigHandler.newConfig = True - for tr in StereoConfigHandler.trSpeckleRange: - tr.set(value) - - def trackbarDecimationFactor(value): - StereoConfigHandler.config.postProcessing.decimationFilter.decimationFactor = value - StereoConfigHandler.newConfig = True - for tr in StereoConfigHandler.trDecimationFactor: - tr.set(value) - - def trackbarDisparityShift(value): - StereoConfigHandler.config.algorithmControl.disparityShift = value - StereoConfigHandler.newConfig = True - for tr in StereoConfigHandler.trDisparityShift: - tr.set(value) - - def trackbarCenterAlignmentShift(value): - if StereoConfigHandler.config.algorithmControl.depthAlign != dai.StereoDepthConfig.AlgorithmControl.DepthAlign.CENTER: - print("Center alignment shift factor requires CENTER alignment enabled!") - return - StereoConfigHandler.config.algorithmControl.centerAlignmentShiftFactor = value / 100. - print(f"centerAlignmentShiftFactor: {StereoConfigHandler.config.algorithmControl.centerAlignmentShiftFactor:.2f}") - StereoConfigHandler.newConfig = True - for tr in StereoConfigHandler.trCenterAlignmentShift: - tr.set(value) - - def trackbarInvalidateEdgePixels(value): - StereoConfigHandler.config.algorithmControl.numInvalidateEdgePixels = value - print(f"numInvalidateEdgePixels: {StereoConfigHandler.config.algorithmControl.numInvalidateEdgePixels:.2f}") - StereoConfigHandler.newConfig = True - for tr in StereoConfigHandler.trInvalidateEdgePixels: - tr.set(value) - - def handleKeypress(key, stereoDepthConfigInQueue): - if key == ord("m"): - StereoConfigHandler.newConfig = True - medianSettings = [dai.MedianFilter.MEDIAN_OFF, dai.MedianFilter.KERNEL_3x3, dai.MedianFilter.KERNEL_5x5, dai.MedianFilter.KERNEL_7x7] - currentMedian = StereoConfigHandler.config.postProcessing.median - nextMedian = medianSettings[(medianSettings.index(currentMedian)+1) % len(medianSettings)] - print(f"Changing median to {nextMedian.name} from {currentMedian.name}") - StereoConfigHandler.config.postProcessing.median = nextMedian - if key == ord("w"): - StereoConfigHandler.newConfig = True - StereoConfigHandler.config.postProcessing.spatialFilter.enable = not StereoConfigHandler.config.postProcessing.spatialFilter.enable - state = "on" if StereoConfigHandler.config.postProcessing.spatialFilter.enable else "off" - print(f"Spatial filter {state}") - if key == ord("t"): - StereoConfigHandler.newConfig = True - StereoConfigHandler.config.postProcessing.temporalFilter.enable = not StereoConfigHandler.config.postProcessing.temporalFilter.enable - state = "on" if StereoConfigHandler.config.postProcessing.temporalFilter.enable else "off" - print(f"Temporal filter {state}") - if key == ord("s"): - StereoConfigHandler.newConfig = True - StereoConfigHandler.config.postProcessing.speckleFilter.enable = not StereoConfigHandler.config.postProcessing.speckleFilter.enable - state = "on" if StereoConfigHandler.config.postProcessing.speckleFilter.enable else "off" - print(f"Speckle filter {state}") - if key == ord("r"): - StereoConfigHandler.newConfig = True - temporalSettings = [dai.StereoDepthConfig.PostProcessing.TemporalFilter.PersistencyMode.PERSISTENCY_OFF, - dai.StereoDepthConfig.PostProcessing.TemporalFilter.PersistencyMode.VALID_8_OUT_OF_8, - dai.StereoDepthConfig.PostProcessing.TemporalFilter.PersistencyMode.VALID_2_IN_LAST_3, - dai.StereoDepthConfig.PostProcessing.TemporalFilter.PersistencyMode.VALID_2_IN_LAST_4, - dai.StereoDepthConfig.PostProcessing.TemporalFilter.PersistencyMode.VALID_2_OUT_OF_8, - dai.StereoDepthConfig.PostProcessing.TemporalFilter.PersistencyMode.VALID_1_IN_LAST_2, - dai.StereoDepthConfig.PostProcessing.TemporalFilter.PersistencyMode.VALID_1_IN_LAST_5, - dai.StereoDepthConfig.PostProcessing.TemporalFilter.PersistencyMode.VALID_1_IN_LAST_8, - dai.StereoDepthConfig.PostProcessing.TemporalFilter.PersistencyMode.PERSISTENCY_INDEFINITELY, - ] - currentTemporal = StereoConfigHandler.config.postProcessing.temporalFilter.persistencyMode - nextTemporal = temporalSettings[(temporalSettings.index(currentTemporal)+1) % len(temporalSettings)] - print(f"Changing temporal persistency to {nextTemporal.name} from {currentTemporal.name}") - StereoConfigHandler.config.postProcessing.temporalFilter.persistencyMode = nextTemporal - if key == ord("n"): - StereoConfigHandler.newConfig = True - decimationSettings = [dai.StereoDepthConfig.PostProcessing.DecimationFilter.DecimationMode.PIXEL_SKIPPING, - dai.StereoDepthConfig.PostProcessing.DecimationFilter.DecimationMode.NON_ZERO_MEDIAN, - dai.StereoDepthConfig.PostProcessing.DecimationFilter.DecimationMode.NON_ZERO_MEAN, - ] - currentDecimation = StereoConfigHandler.config.postProcessing.decimationFilter.decimationMode - nextDecimation = decimationSettings[(decimationSettings.index(currentDecimation)+1) % len(decimationSettings)] - print(f"Changing decimation mode to {nextDecimation.name} from {currentDecimation.name}") - StereoConfigHandler.config.postProcessing.decimationFilter.decimationMode = nextDecimation - if key == ord("a"): - StereoConfigHandler.newConfig = True - alignmentSettings = [dai.StereoDepthConfig.AlgorithmControl.DepthAlign.RECTIFIED_RIGHT, - dai.StereoDepthConfig.AlgorithmControl.DepthAlign.RECTIFIED_LEFT, - dai.StereoDepthConfig.AlgorithmControl.DepthAlign.CENTER, - ] - currentAlignment = StereoConfigHandler.config.algorithmControl.depthAlign - nextAlignment = alignmentSettings[(alignmentSettings.index(currentAlignment)+1) % len(alignmentSettings)] - print(f"Changing alignment mode to {nextAlignment.name} from {currentAlignment.name}") - StereoConfigHandler.config.algorithmControl.depthAlign = nextAlignment - elif key == ord("c"): - StereoConfigHandler.newConfig = True - censusSettings = [dai.StereoDepthConfig.CensusTransform.KernelSize.AUTO, dai.StereoDepthConfig.CensusTransform.KernelSize.KERNEL_5x5, dai.StereoDepthConfig.CensusTransform.KernelSize.KERNEL_7x7, dai.StereoDepthConfig.CensusTransform.KernelSize.KERNEL_7x9] - currentCensus = StereoConfigHandler.config.censusTransform.kernelSize - nextCensus = censusSettings[(censusSettings.index(currentCensus)+1) % len(censusSettings)] - if nextCensus != dai.StereoDepthConfig.CensusTransform.KernelSize.AUTO: - censusGridSize = [(5,5), (7,7), (7,9)] - censusDefaultMask = [np.uint64(0XA82415), np.uint64(0XAA02A8154055), np.uint64(0X2AA00AA805540155)] - censusGrid = censusGridSize[nextCensus] - censusMask = censusDefaultMask[nextCensus] - StereoConfigHandler.censusMaskHandler = StereoConfigHandler.CensusMaskHandler("Census mask", censusGrid) - StereoConfigHandler.censusMaskHandler.setMask(censusMask) - else: - print("Census mask config is not available in AUTO census kernel mode. Change using the 'c' key") - StereoConfigHandler.config.censusTransform.kernelMask = 0 - StereoConfigHandler.censusMaskHandler.destroyWindow() - print(f"Changing census transform to {nextCensus.name} from {currentCensus.name}") - StereoConfigHandler.config.censusTransform.kernelSize = nextCensus - elif key == ord("d"): - StereoConfigHandler.newConfig = True - dispRangeSettings = [dai.StereoDepthConfig.CostMatching.DisparityWidth.DISPARITY_64, dai.StereoDepthConfig.CostMatching.DisparityWidth.DISPARITY_96] - currentDispRange = StereoConfigHandler.config.costMatching.disparityWidth - nextDispRange = dispRangeSettings[(dispRangeSettings.index(currentDispRange)+1) % len(dispRangeSettings)] - print(f"Changing disparity range to {nextDispRange.name} from {currentDispRange.name}") - StereoConfigHandler.config.costMatching.disparityWidth = nextDispRange - elif key == ord("f"): - StereoConfigHandler.newConfig = True - StereoConfigHandler.config.costMatching.enableCompanding = not StereoConfigHandler.config.costMatching.enableCompanding - state = "on" if StereoConfigHandler.config.costMatching.enableCompanding else "off" - print(f"Companding {state}") - elif key == ord("v"): - StereoConfigHandler.newConfig = True - StereoConfigHandler.config.censusTransform.enableMeanMode = not StereoConfigHandler.config.censusTransform.enableMeanMode - state = "on" if StereoConfigHandler.config.censusTransform.enableMeanMode else "off" - print(f"Census transform mean mode {state}") - elif key == ord("1"): - StereoConfigHandler.newConfig = True - StereoConfigHandler.config.algorithmControl.enableLeftRightCheck = not StereoConfigHandler.config.algorithmControl.enableLeftRightCheck - state = "on" if StereoConfigHandler.config.algorithmControl.enableLeftRightCheck else "off" - print(f"LR-check {state}") - elif key == ord("2"): - StereoConfigHandler.newConfig = True - StereoConfigHandler.config.algorithmControl.enableSubpixel = not StereoConfigHandler.config.algorithmControl.enableSubpixel - state = "on" if StereoConfigHandler.config.algorithmControl.enableSubpixel else "off" - print(f"Subpixel {state}") - elif key == ord("3"): - StereoConfigHandler.newConfig = True - StereoConfigHandler.config.algorithmControl.enableExtended = not StereoConfigHandler.config.algorithmControl.enableExtended - state = "on" if StereoConfigHandler.config.algorithmControl.enableExtended else "off" - print(f"Extended {state}") - - censusMaskChanged = False - if StereoConfigHandler.censusMaskHandler is not None: - censusMaskChanged = StereoConfigHandler.censusMaskHandler.isChanged() - if censusMaskChanged: - StereoConfigHandler.config.censusTransform.kernelMask = StereoConfigHandler.censusMaskHandler.getMask() - StereoConfigHandler.newConfig = True - - StereoConfigHandler.sendConfig(stereoDepthConfigInQueue) - - def sendConfig(stereoDepthConfigInQueue): - if StereoConfigHandler.newConfig: - StereoConfigHandler.newConfig = False - configMessage = dai.StereoDepthConfig() - configMessage.set(StereoConfigHandler.config) - stereoDepthConfigInQueue.send(configMessage) - - def updateDefaultConfig(config): - StereoConfigHandler.config = config - - def registerWindow(stream): - cv2.namedWindow(stream, cv2.WINDOW_NORMAL) - - StereoConfigHandler.trConfidence.append(StereoConfigHandler.Trackbar("Disparity confidence", stream, 0, 255, StereoConfigHandler.config.costMatching.confidenceThreshold, StereoConfigHandler.trackbarConfidence)) - StereoConfigHandler.trSigma.append(StereoConfigHandler.Trackbar("Bilateral sigma", stream, 0, 100, StereoConfigHandler.config.postProcessing.bilateralSigmaValue, StereoConfigHandler.trackbarSigma)) - StereoConfigHandler.trLrCheck.append(StereoConfigHandler.Trackbar("LR-check threshold", stream, 0, 16, StereoConfigHandler.config.algorithmControl.leftRightCheckThreshold, StereoConfigHandler.trackbarLrCheckThreshold)) - StereoConfigHandler.trFractionalBits.append(StereoConfigHandler.Trackbar("Subpixel fractional bits", stream, 3, 5, StereoConfigHandler.config.algorithmControl.subpixelFractionalBits, StereoConfigHandler.trackbarFractionalBits)) - StereoConfigHandler.trDisparityShift.append(StereoConfigHandler.Trackbar("Disparity shift", stream, 0, 100, StereoConfigHandler.config.algorithmControl.disparityShift, StereoConfigHandler.trackbarDisparityShift)) - StereoConfigHandler.trCenterAlignmentShift.append(StereoConfigHandler.Trackbar("Center alignment shift factor", stream, 0, 100, StereoConfigHandler.config.algorithmControl.centerAlignmentShiftFactor, StereoConfigHandler.trackbarCenterAlignmentShift)) - StereoConfigHandler.trInvalidateEdgePixels.append(StereoConfigHandler.Trackbar("Invalidate edge pixels", stream, 0, 100, StereoConfigHandler.config.algorithmControl.numInvalidateEdgePixels, StereoConfigHandler.trackbarInvalidateEdgePixels)) - StereoConfigHandler.trLineqAlpha.append(StereoConfigHandler.Trackbar("Linear equation alpha", stream, 0, 15, StereoConfigHandler.config.costMatching.linearEquationParameters.alpha, StereoConfigHandler.trackbarLineqAlpha)) - StereoConfigHandler.trLineqBeta.append(StereoConfigHandler.Trackbar("Linear equation beta", stream, 0, 15, StereoConfigHandler.config.costMatching.linearEquationParameters.beta, StereoConfigHandler.trackbarLineqBeta)) - StereoConfigHandler.trLineqThreshold.append(StereoConfigHandler.Trackbar("Linear equation threshold", stream, 0, 255, StereoConfigHandler.config.costMatching.linearEquationParameters.threshold, StereoConfigHandler.trackbarLineqThreshold)) - StereoConfigHandler.trCostAggregationP1.append(StereoConfigHandler.Trackbar("Cost aggregation P1", stream, 0, 500, StereoConfigHandler.config.costAggregation.horizontalPenaltyCostP1, StereoConfigHandler.trackbarCostAggregationP1)) - StereoConfigHandler.trCostAggregationP2.append(StereoConfigHandler.Trackbar("Cost aggregation P2", stream, 0, 500, StereoConfigHandler.config.costAggregation.horizontalPenaltyCostP2, StereoConfigHandler.trackbarCostAggregationP2)) - StereoConfigHandler.trTemporalAlpha.append(StereoConfigHandler.Trackbar("Temporal filter alpha", stream, 0, 100, int(StereoConfigHandler.config.postProcessing.temporalFilter.alpha*100), StereoConfigHandler.trackbarTemporalFilterAlpha)) - StereoConfigHandler.trTemporalDelta.append(StereoConfigHandler.Trackbar("Temporal filter delta", stream, 0, 100, StereoConfigHandler.config.postProcessing.temporalFilter.delta, StereoConfigHandler.trackbarTemporalFilterDelta)) - StereoConfigHandler.trSpatialAlpha.append(StereoConfigHandler.Trackbar("Spatial filter alpha", stream, 0, 100, int(StereoConfigHandler.config.postProcessing.spatialFilter.alpha*100), StereoConfigHandler.trackbarSpatialFilterAlpha)) - StereoConfigHandler.trSpatialDelta.append(StereoConfigHandler.Trackbar("Spatial filter delta", stream, 0, 100, StereoConfigHandler.config.postProcessing.spatialFilter.delta, StereoConfigHandler.trackbarSpatialFilterDelta)) - StereoConfigHandler.trSpatialHoleFilling.append(StereoConfigHandler.Trackbar("Spatial filter hole filling radius", stream, 0, 16, StereoConfigHandler.config.postProcessing.spatialFilter.holeFillingRadius, StereoConfigHandler.trackbarSpatialFilterHoleFillingRadius)) - StereoConfigHandler.trSpatialNumIterations.append(StereoConfigHandler.Trackbar("Spatial filter number of iterations", stream, 0, 4, StereoConfigHandler.config.postProcessing.spatialFilter.numIterations, StereoConfigHandler.trackbarSpatialFilterNumIterations)) - StereoConfigHandler.trThresholdMinRange.append(StereoConfigHandler.Trackbar("Threshold filter min range", stream, 0, 65, StereoConfigHandler.config.postProcessing.thresholdFilter.minRange, StereoConfigHandler.trackbarThresholdMinRange)) - StereoConfigHandler.trThresholdMaxRange.append(StereoConfigHandler.Trackbar("Threshold filter max range", stream, 0, 65, StereoConfigHandler.config.postProcessing.thresholdFilter.maxRange, StereoConfigHandler.trackbarThresholdMaxRange)) - StereoConfigHandler.trSpeckleRange.append(StereoConfigHandler.Trackbar("Speckle filter range", stream, 0, 240, StereoConfigHandler.config.postProcessing.speckleFilter.speckleRange, StereoConfigHandler.trackbarSpeckleRange)) - StereoConfigHandler.trDecimationFactor.append(StereoConfigHandler.Trackbar("Decimation factor", stream, 1, 4, StereoConfigHandler.config.postProcessing.decimationFilter.decimationFactor, StereoConfigHandler.trackbarDecimationFactor)) - - def __init__(self, config): - print("Control median filter using the 'm' key.") - print("Control census transform kernel size using the 'c' key.") - print("Control disparity search range using the 'd' key.") - print("Control disparity companding using the 'f' key.") - print("Control census transform mean mode using the 'v' key.") - print("Control depth alignment using the 'a' key.") - print("Control decimation algorithm using the 'a' key.") - print("Control temporal persistency mode using the 'r' key.") - print("Control spatial filter using the 'w' key.") - print("Control temporal filter using the 't' key.") - print("Control speckle filter using the 's' key.") - print("Control left-right check mode using the '1' key.") - print("Control subpixel mode using the '2' key.") - print("Control extended mode using the '3' key.") - - StereoConfigHandler.config = config - - if StereoConfigHandler.config.censusTransform.kernelSize != dai.StereoDepthConfig.CensusTransform.KernelSize.AUTO: - censusMask = StereoConfigHandler.config.censusTransform.kernelMask - censusGridSize = [(5,5), (7,7), (7,9)] - censusGrid = censusGridSize[StereoConfigHandler.config.censusTransform.kernelSize] - if StereoConfigHandler.config.censusTransform.kernelMask == 0: - censusDefaultMask = [np.uint64(0xA82415), np.uint64(0xAA02A8154055), np.uint64(0x2AA00AA805540155)] - censusMask = censusDefaultMask[StereoConfigHandler.config.censusTransform.kernelSize] - StereoConfigHandler.censusMaskHandler = StereoConfigHandler.CensusMaskHandler("Census mask", censusGrid) - StereoConfigHandler.censusMaskHandler.setMask(censusMask) - else: - print("Census mask config is not available in AUTO Census kernel mode. Change using the 'c' key") - - -# StereoDepth initial config options. -outDepth = True # Disparity by default -outConfidenceMap = True # Output disparity confidence map -outRectified = True # Output and display rectified streams -lrcheck = True # Better handling for occlusions -extended = False # Closer-in minimum depth, disparity range is doubled. Unsupported for now. -subpixel = True # Better accuracy for longer distance, fractional disparity 32-levels - -width = 1280 -height = 800 - -xoutStereoCfg = None - -# Create pipeline -pipeline = dai.Pipeline() - -# Define sources and outputs -stereo = pipeline.create(dai.node.StereoDepth) - -monoLeft = pipeline.create(dai.node.XLinkIn) -monoRight = pipeline.create(dai.node.XLinkIn) -xinStereoDepthConfig = pipeline.create(dai.node.XLinkIn) - -xoutLeft = pipeline.create(dai.node.XLinkOut) -xoutRight = pipeline.create(dai.node.XLinkOut) -xoutDepth = pipeline.create(dai.node.XLinkOut) -xoutConfMap = pipeline.create(dai.node.XLinkOut) -xoutDisparity = pipeline.create(dai.node.XLinkOut) -xoutRectifLeft = pipeline.create(dai.node.XLinkOut) -xoutRectifRight = pipeline.create(dai.node.XLinkOut) -xoutStereoCfg = pipeline.create(dai.node.XLinkOut) -if args.debug: - xoutDebugLrCheckIt1 = pipeline.create(dai.node.XLinkOut) - xoutDebugLrCheckIt2 = pipeline.create(dai.node.XLinkOut) - xoutDebugExtLrCheckIt1 = pipeline.create(dai.node.XLinkOut) - xoutDebugExtLrCheckIt2 = pipeline.create(dai.node.XLinkOut) -if args.dumpdisparitycostvalues: - xoutDebugCostDump = pipeline.create(dai.node.XLinkOut) - -xinStereoDepthConfig.setStreamName("stereoDepthConfig") -monoLeft.setStreamName("in_left") -monoRight.setStreamName("in_right") - -xoutLeft.setStreamName("left") -xoutRight.setStreamName("right") -xoutDepth.setStreamName("depth") -xoutConfMap.setStreamName("confidence_map") -xoutDisparity.setStreamName("disparity") -xoutRectifLeft.setStreamName("rectified_left") -xoutRectifRight.setStreamName("rectified_right") -xoutStereoCfg.setStreamName("stereo_cfg") -if args.debug: - xoutDebugLrCheckIt1.setStreamName("disparity_lr_check_iteration1") - xoutDebugLrCheckIt2.setStreamName("disparity_lr_check_iteration2") - xoutDebugExtLrCheckIt1.setStreamName("disparity_ext_lr_check_iteration1") - xoutDebugExtLrCheckIt2.setStreamName("disparity_ext_lr_check_iteration2") -if args.dumpdisparitycostvalues: - xoutDebugCostDump.setStreamName("disparity_cost_dump") - -# Properties -stereo.setDefaultProfilePreset(dai.node.StereoDepth.PresetMode.HIGH_DENSITY) -stereo.setRectifyEdgeFillColor(0) # Black, to better see the cutout -stereo.setLeftRightCheck(lrcheck) -stereo.setExtendedDisparity(extended) -stereo.setSubpixel(subpixel) - -# Switching depthAlign mode at runtime is not supported while aligning to a specific camera is enabled -# stereo.setDepthAlign(dai.CameraBoardSocket.LEFT) - -# allocates resources for worst case scenario -# allowing runtime switch of stereo modes -stereo.setRuntimeModeSwitch(True) - -# Linking -monoLeft.out.link(stereo.left) -monoRight.out.link(stereo.right) -xinStereoDepthConfig.out.link(stereo.inputConfig) -stereo.syncedLeft.link(xoutLeft.input) -stereo.syncedRight.link(xoutRight.input) -if outDepth: - stereo.depth.link(xoutDepth.input) -if outConfidenceMap: - stereo.confidenceMap.link(xoutConfMap.input) -stereo.disparity.link(xoutDisparity.input) -if outRectified: - stereo.rectifiedLeft.link(xoutRectifLeft.input) - stereo.rectifiedRight.link(xoutRectifRight.input) -stereo.outConfig.link(xoutStereoCfg.input) -if args.debug: - stereo.debugDispLrCheckIt1.link(xoutDebugLrCheckIt1.input) - stereo.debugDispLrCheckIt2.link(xoutDebugLrCheckIt2.input) - stereo.debugExtDispLrCheckIt1.link(xoutDebugExtLrCheckIt1.input) - stereo.debugExtDispLrCheckIt2.link(xoutDebugExtLrCheckIt2.input) -if args.dumpdisparitycostvalues: - stereo.debugDispCostDump.link(xoutDebugCostDump.input) - - -StereoConfigHandler(stereo.initialConfig.get()) -StereoConfigHandler.registerWindow("Stereo control panel") - -# stereo.setPostProcessingHardwareResources(3, 3) - -stereo.setInputResolution(width, height) -stereo.setRectification(False) -baseline = 75 -fov = 71.86 -focal = width / (2 * math.tan(fov / 2 / 180 * math.pi)) - -streams = ["left", "right"] -if outRectified: - streams.extend(["rectified_left", "rectified_right"]) -streams.append("disparity") -if outDepth: - streams.append("depth") -if outConfidenceMap: - streams.append("confidence_map") -debugStreams = [] -if args.debug: - debugStreams.extend(["disparity_lr_check_iteration1", "disparity_lr_check_iteration2"]) - debugStreams.extend(["disparity_ext_lr_check_iteration1", "disparity_ext_lr_check_iteration2"]) -if args.dumpdisparitycostvalues: - debugStreams.append("disparity_cost_dump") - -def convertToCv2Frame(name, image, config): - - maxDisp = config.getMaxDisparity() - subpixelLevels = pow(2, config.get().algorithmControl.subpixelFractionalBits) - subpixel = config.get().algorithmControl.enableSubpixel - dispIntegerLevels = maxDisp if not subpixel else maxDisp / subpixelLevels - - frame = image.getFrame() - - # frame.tofile(name+".raw") - - if name == "depth": - dispScaleFactor = baseline * focal - with np.errstate(divide="ignore"): - frame = dispScaleFactor / frame - - frame = (frame * 255. / dispIntegerLevels).astype(np.uint8) - frame = cv2.applyColorMap(frame, cv2.COLORMAP_HOT) - elif "confidence_map" in name: - pass - elif name == "disparity_cost_dump": - # frame.tofile(name+".raw") - pass - elif "disparity" in name: - if 1: # Optionally, extend disparity range to better visualize it - frame = (frame * 255. / maxDisp).astype(np.uint8) - return frame - # if 1: # Optionally, apply a color map - # frame = cv2.applyColorMap(frame, cv2.COLORMAP_HOT) - - return frame - -class DatasetManager: - def __init__(self, path): - self.path = path - self.index = 0 - self.names = [d for d in os.listdir(path) if os.path.isdir(os.path.join(path, d))] - if len(self.names) == 0: - raise RuntimeError("No dataset found at {}".format(path)) - - def get(self): - return os.path.join(self.path, self.names[self.index]) - - def get_name(self): - return self.names[self.index] - - def next(self): - self.index = (self.index + 1) % len(self.names) - return self.get() - - def prev(self): - self.index = (self.index - 1) % len(self.names) - return self.get() - - -def calculate_err_measures(gt_img, oak_img): - assert gt_img.shape == oak_img.shape - - gt_mask = gt_img != np.inf - oak_mask = oak_img != np.inf - mask = gt_mask & oak_mask - - gt_img[~gt_mask] = 0. - oak_img[~mask] = 0. - err = np.abs(gt_img - oak_img) - - n = np.sum(gt_mask) - invalid = np.sum(gt_mask & ~oak_mask) - - bad05 = np.sum(mask & (err > 0.5)) - bad1 = np.sum(mask & (err > 1.)) - bad2 = np.sum(mask & (err > 2.)) - bad4 = np.sum(mask & (err > 4.)) - sum_err = np.sum(err[mask]) - sum_sq_err = np.sum(err[mask] ** 2) - errs = err[mask] - - bad05_p = 100. * bad05 / n - total_bad05_p = 100. * (bad05 + invalid) / n - bad1_p = 100. * bad1 / n - total_bad1_p = 100. * (bad1 + invalid) / n - bad2_p = 100. * bad2 / n - total_bad2_p = 100. * (bad2 + invalid) / n - bad4_p = 100. * bad4 / n - total_bad4_p = 100. * (bad4 + invalid) / n - invalid_p = 100. * invalid / n - avg_err = sum_err / (n - invalid) - mse = sum_sq_err / (n - invalid) - a50 = np.percentile(errs, 50) - a90 = np.percentile(errs, 90) - a95 = np.percentile(errs, 95) - a99 = np.percentile(errs, 99) - - return { - "bad0.5": bad05_p, - "total_bad0.5": total_bad05_p, - "bad1": bad1_p, - "total_bad1": total_bad1_p, - "bad2": bad2_p, - "total_bad2": total_bad2_p, - "bad4": bad4_p, - "total_bad4": total_bad4_p, - "invalid": invalid_p, - "avg_err": avg_err, - "mse": mse, - "a50": a50, - "a90": a90, - "a95": a95, - "a99": a99 - } - -def show_evaluation(img_name, evals): - cv2.namedWindow("Evaluation", cv2.WINDOW_NORMAL) - font = cv2.FONT_HERSHEY_SIMPLEX - font_scale = 2 - thickness = 3 - color = (0, 0, 0) - lines = [ - f"Name: {img_name}", - f"Bad0.5: {evals['bad0.5']:.2f}%", - f"Total Bad0.5: {evals['total_bad0.5']:.2f}%", - f"Bad1: {evals['bad1']:.2f}%", - f"Total Bad1: {evals['total_bad1']:.2f}%", - f"Bad2: {evals['bad2']:.2f}%", - f"Total Bad2: {evals['total_bad2']:.2f}%", - f"Bad4: {evals['bad4']:.2f}%", - f"Total Bad4: {evals['total_bad4']:.2f}%", - f"Invalid: {evals['invalid']:.2f}%", - f"Avg Err: {evals['avg_err']:.2f}", - f"MSE: {evals['mse']:.2f}", - f"A50: {evals['a50']:.2f}", - f"A90: {evals['a90']:.2f}", - f"A95: {evals['a95']:.2f}", - f"A99: {evals['a99']:.2f}" - ] - sizes = [cv2.getTextSize(line, font, font_scale, thickness) for line in lines] - sizes = [(size[0][0], size[0][1] + size[1], size[1]) for size in sizes] - max_width = max([size[0] for size in sizes]) - total_height = sum([size[1] for size in sizes]) + (len(lines) - 1) * thickness - img = np.ones((total_height + thickness, max_width, 3), dtype=np.uint8) * 255 - y = 0 - for line, size in zip(lines, sizes): - cv2.putText(img, line, (0, y + size[1] - size[2]), font, font_scale, color, thickness) - y += size[1] + thickness - cv2.imshow("Evaluation", img) - -def show_debug_disparity(gt_img, oak_img): - def rescale_img(img): - img[img == np.inf] = 0. - img = cv2.resize(img, (1280, 800), interpolation=cv2.INTER_AREA) - return img.astype(np.uint16) - - gt_img = rescale_img(gt_img) - oak_img = rescale_img(oak_img) - maxv = max(gt_img.max(), oak_img.max()) - gt_img = (gt_img * 255. / maxv).astype(np.uint8) - oak_img = (oak_img * 255. / maxv).astype(np.uint8) - cv2.imshow("GT", gt_img) - cv2.imshow("OAK", oak_img) - -dataset = DatasetManager(args.dataset) - -print("Connecting and starting the pipeline") -# Connect to device and start pipeline -with dai.Device(pipeline) as device: - - stereoDepthConfigInQueue = device.getInputQueue("stereoDepthConfig") - inStreams = ["in_left", "in_right"] - inStreamsCameraID = [dai.CameraBoardSocket.CAM_B, dai.CameraBoardSocket.CAM_C] - in_q_list = [] - for s in inStreams: - q = device.getInputQueue(s) - in_q_list.append(q) - - # Create a receive queue for each stream - q_list = [] - for s in streams: - q = device.getOutputQueue(s, 8, blocking=False) - q_list.append(q) - - inCfg = device.getOutputQueue("stereo_cfg", 8, blocking=False) - - # Need to set a timestamp for input frames, for the sync stage in Stereo node - timestamp_ms = 0 - prevQueues = q_list.copy() - while True: - # Handle input streams, if any - if in_q_list: - dataset_size = 1 # Number of image pairs - frame_interval_ms = 50 - for i, q in enumerate(in_q_list): - path = os.path.join(dataset.get(), f"im{i}.png") - data = cv2.imread(path, cv2.IMREAD_GRAYSCALE) - data = cv2.resize(data, (width, height), interpolation = cv2.INTER_AREA) - data = data.reshape(height*width) - tstamp = datetime.timedelta(seconds = timestamp_ms // 1000, - milliseconds = timestamp_ms % 1000) - img = dai.ImgFrame() - img.setData(data) - img.setTimestamp(tstamp) - img.setInstanceNum(inStreamsCameraID[i]) - img.setType(dai.ImgFrame.Type.RAW8) - img.setWidth(width) - img.setHeight(height) - q.send(img) - # print("Sent frame: {:25s}".format(path), "timestamp_ms:", timestamp_ms) - timestamp_ms += frame_interval_ms - sleep(frame_interval_ms / 1000) - - # Load GT disparity - gt_disparity = None - if args.evaluate: - gt_disparity = read_pfm(os.path.join(dataset.get(), f"disp0.pfm"))[0] - - # Handle output streams - currentConfig = inCfg.get() - - lrCheckEnabled = currentConfig.get().algorithmControl.enableLeftRightCheck - extendedEnabled = currentConfig.get().algorithmControl.enableExtended - queues = q_list.copy() - - if args.dumpdisparitycostvalues: - q = device.getOutputQueue("disparity_cost_dump", 8, blocking=False) - queues.append(q) - - if args.debug: - q_list_debug = [] - - activeDebugStreams = [] - if lrCheckEnabled: - activeDebugStreams.extend(["disparity_lr_check_iteration1", "disparity_lr_check_iteration2"]) - if extendedEnabled: - activeDebugStreams.extend(["disparity_ext_lr_check_iteration1"]) - if lrCheckEnabled: - activeDebugStreams.extend(["disparity_ext_lr_check_iteration2"]) - - for s in activeDebugStreams: - q = device.getOutputQueue(s, 8, blocking=False) - q_list_debug.append(q) - - queues.extend(q_list_debug) - - def ListDiff(li1, li2): - return list(set(li1) - set(li2)) + list(set(li2) - set(li1)) - - diff = ListDiff(prevQueues, queues) - for s in diff: - name = s.getName() - cv2.destroyWindow(name) - prevQueues = queues.copy() - - disparity = None - for q in queues: - if q.getName() in ["left", "right"]: continue - data = q.get() - if q.getName() == "disparity": - disparity = data.getFrame() - frame = convertToCv2Frame(q.getName(), data, currentConfig) - cv2.imshow(q.getName(), frame) - - if disparity is not None and gt_disparity is not None: - subpixel_bits = 1 << currentConfig.get().algorithmControl.subpixelFractionalBits - subpixel_enabled = currentConfig.get().algorithmControl.enableSubpixel - width_scale = float(gt_disparity.shape[1]) / float(disparity.shape[1]) - - disparity = disparity.astype(np.float32) - if subpixel_enabled: - disparity = disparity / subpixel_bits - disparity = disparity * width_scale - disparity = cv2.resize(disparity, (gt_disparity.shape[1], gt_disparity.shape[0]), interpolation = cv2.INTER_LINEAR) - disparity[disparity == 0.] = np.inf - - # show_debug_disparity(gt_disparity, disparity) - err_vals = calculate_err_measures(gt_disparity, disparity) - show_evaluation(dataset.get_name(), err_vals) - - key = cv2.waitKey(1) - if key == ord("q"): - break - elif key == ord("["): - dataset.next() - elif key == ord("]"): - dataset.prev() - - StereoConfigHandler.handleKeypress(key, stereoDepthConfigInQueue) diff --git a/examples/StereoDepth/stereo_depth_from_host.py b/examples/StereoDepth/stereo_depth_from_host.py index bf5328785..7c6225541 100755 --- a/examples/StereoDepth/stereo_depth_from_host.py +++ b/examples/StereoDepth/stereo_depth_from_host.py @@ -8,19 +8,62 @@ import argparse from pathlib import Path import math +import os, re -datasetDefault = str((Path(__file__).parent / Path('../models/dataset')).resolve().absolute()) +datasetDefault = str((Path(__file__).parent / Path("../models/dataset")).resolve().absolute()) parser = argparse.ArgumentParser() -parser.add_argument('-dataset', nargs='?', help="Path to recorded frames", default=datasetDefault) -parser.add_argument('-debug', "--debug", action="store_true", help="Enable debug outputs.") -parser.add_argument('-dumpdispcost', "--dumpdisparitycostvalues", action="store_true", help="Dumps the disparity cost values for each disparity range. 96 byte for each pixel.") +parser.add_argument("-p", "--dataset", nargs="?", help="Path to recorded frames", default=None) +parser.add_argument("-d", "--debug", action="store_true", help="Enable debug outputs.") +parser.add_argument("-e", "--evaluate", help="Evaluate the disparity calculation.", default=None) +parser.add_argument("-dumpdispcost", "--dumpdisparitycostvalues", action="store_true", help="Dumps the disparity cost values for each disparity range. 96 byte for each pixel.") +parser.add_argument("--download", action="store_true", help="Downloads the 2014 Middlebury dataset.") args = parser.parse_args() -if not Path(datasetDefault).exists(): +if args.evaluate is not None and args.dataset is not None: import sys - raise FileNotFoundError(f'Required file/s not found, please run "{sys.executable} install_requirements.py"') + raise ValueError("Cannot use both --dataset and --evaluate arguments at the same time.") +evaluation_mode = args.evaluate is not None +args.dataset = args.dataset or datasetDefault +if args.download and args.evaluate is None: + import sys + raise ValueError("Cannot use --download without --evaluate argument.") + +if args.evaluate is None and not Path(args.dataset).exists(): + import sys + raise FileNotFoundError(f"Required file/s not found, please run '{sys.executable} install_requirements.py'") + +if args.evaluate is not None and not args.download and not Path(args.evaluate).exists(): + import sys + raise FileNotFoundError(f"Evaluation dataset path does not exist, use the --evaluate argument to specify the path.") + +if args.evaluate is not None and args.download and not Path(args.evaluate).exists(): + os.makedirs(args.evaluate) + +def download_2014_middlebury(data_path): + import requests, zipfile, io + url = "https://vision.middlebury.edu/stereo/data/scenes2014/zip/" + r = requests.get(url) + c = r.content + reg = re.compile(r"href=('|\")(.+\.zip)('|\")") + matches = reg.findall(c.decode("utf-8")) + files = [m[1] for m in matches] + + for f in files: + if os.path.isdir(os.path.join(data_path, f[:-4])): + print(f"Skipping {f}") + else: + print(f"Downloading {f} from {url + f}") + r = requests.get(url + f) + print(f"Extracting {f} to {data_path}") + z = zipfile.ZipFile(io.BytesIO(r.content)) + z.extractall(data_path) + +if args.download: + download_2014_middlebury(args.evaluate) + if not evaluation_mode: + sys.exit(0) class StereoConfigHandler: @@ -36,10 +79,10 @@ def __init__(self, trackbarName, windowName, minValue, maxValue, defaultValue, h def set(self, value): if value < self.min: value = self.min - print(f'{self.trackbarName} min value is {self.min}') + print(f"{self.trackbarName} min value is {self.min}") if value > self.max: value = self.max - print(f'{self.trackbarName} max value is {self.max}') + print(f"{self.trackbarName} max value is {self.max}") cv2.setTrackbarPos(self.trackbarName, self.windowName, value) class CensusMaskHandler: @@ -299,29 +342,29 @@ def trackbarInvalidateEdgePixels(value): tr.set(value) def handleKeypress(key, stereoDepthConfigInQueue): - if key == ord('m'): + if key == ord("m"): StereoConfigHandler.newConfig = True medianSettings = [dai.MedianFilter.MEDIAN_OFF, dai.MedianFilter.KERNEL_3x3, dai.MedianFilter.KERNEL_5x5, dai.MedianFilter.KERNEL_7x7] currentMedian = StereoConfigHandler.config.postProcessing.median nextMedian = medianSettings[(medianSettings.index(currentMedian)+1) % len(medianSettings)] print(f"Changing median to {nextMedian.name} from {currentMedian.name}") StereoConfigHandler.config.postProcessing.median = nextMedian - if key == ord('w'): + if key == ord("w"): StereoConfigHandler.newConfig = True StereoConfigHandler.config.postProcessing.spatialFilter.enable = not StereoConfigHandler.config.postProcessing.spatialFilter.enable state = "on" if StereoConfigHandler.config.postProcessing.spatialFilter.enable else "off" print(f"Spatial filter {state}") - if key == ord('t'): + if key == ord("t"): StereoConfigHandler.newConfig = True StereoConfigHandler.config.postProcessing.temporalFilter.enable = not StereoConfigHandler.config.postProcessing.temporalFilter.enable state = "on" if StereoConfigHandler.config.postProcessing.temporalFilter.enable else "off" print(f"Temporal filter {state}") - if key == ord('s'): + if key == ord("s"): StereoConfigHandler.newConfig = True StereoConfigHandler.config.postProcessing.speckleFilter.enable = not StereoConfigHandler.config.postProcessing.speckleFilter.enable state = "on" if StereoConfigHandler.config.postProcessing.speckleFilter.enable else "off" print(f"Speckle filter {state}") - if key == ord('r'): + if key == ord("r"): StereoConfigHandler.newConfig = True temporalSettings = [dai.StereoDepthConfig.PostProcessing.TemporalFilter.PersistencyMode.PERSISTENCY_OFF, dai.StereoDepthConfig.PostProcessing.TemporalFilter.PersistencyMode.VALID_8_OUT_OF_8, @@ -337,7 +380,7 @@ def handleKeypress(key, stereoDepthConfigInQueue): nextTemporal = temporalSettings[(temporalSettings.index(currentTemporal)+1) % len(temporalSettings)] print(f"Changing temporal persistency to {nextTemporal.name} from {currentTemporal.name}") StereoConfigHandler.config.postProcessing.temporalFilter.persistencyMode = nextTemporal - if key == ord('n'): + if key == ord("n"): StereoConfigHandler.newConfig = True decimationSettings = [dai.StereoDepthConfig.PostProcessing.DecimationFilter.DecimationMode.PIXEL_SKIPPING, dai.StereoDepthConfig.PostProcessing.DecimationFilter.DecimationMode.NON_ZERO_MEDIAN, @@ -347,7 +390,7 @@ def handleKeypress(key, stereoDepthConfigInQueue): nextDecimation = decimationSettings[(decimationSettings.index(currentDecimation)+1) % len(decimationSettings)] print(f"Changing decimation mode to {nextDecimation.name} from {currentDecimation.name}") StereoConfigHandler.config.postProcessing.decimationFilter.decimationMode = nextDecimation - if key == ord('a'): + if key == ord("a"): StereoConfigHandler.newConfig = True alignmentSettings = [dai.StereoDepthConfig.AlgorithmControl.DepthAlign.RECTIFIED_RIGHT, dai.StereoDepthConfig.AlgorithmControl.DepthAlign.RECTIFIED_LEFT, @@ -357,7 +400,7 @@ def handleKeypress(key, stereoDepthConfigInQueue): nextAlignment = alignmentSettings[(alignmentSettings.index(currentAlignment)+1) % len(alignmentSettings)] print(f"Changing alignment mode to {nextAlignment.name} from {currentAlignment.name}") StereoConfigHandler.config.algorithmControl.depthAlign = nextAlignment - elif key == ord('c'): + elif key == ord("c"): StereoConfigHandler.newConfig = True censusSettings = [dai.StereoDepthConfig.CensusTransform.KernelSize.AUTO, dai.StereoDepthConfig.CensusTransform.KernelSize.KERNEL_5x5, dai.StereoDepthConfig.CensusTransform.KernelSize.KERNEL_7x7, dai.StereoDepthConfig.CensusTransform.KernelSize.KERNEL_7x9] currentCensus = StereoConfigHandler.config.censusTransform.kernelSize @@ -375,34 +418,34 @@ def handleKeypress(key, stereoDepthConfigInQueue): StereoConfigHandler.censusMaskHandler.destroyWindow() print(f"Changing census transform to {nextCensus.name} from {currentCensus.name}") StereoConfigHandler.config.censusTransform.kernelSize = nextCensus - elif key == ord('d'): + elif key == ord("d"): StereoConfigHandler.newConfig = True dispRangeSettings = [dai.StereoDepthConfig.CostMatching.DisparityWidth.DISPARITY_64, dai.StereoDepthConfig.CostMatching.DisparityWidth.DISPARITY_96] currentDispRange = StereoConfigHandler.config.costMatching.disparityWidth nextDispRange = dispRangeSettings[(dispRangeSettings.index(currentDispRange)+1) % len(dispRangeSettings)] print(f"Changing disparity range to {nextDispRange.name} from {currentDispRange.name}") StereoConfigHandler.config.costMatching.disparityWidth = nextDispRange - elif key == ord('f'): + elif key == ord("f"): StereoConfigHandler.newConfig = True StereoConfigHandler.config.costMatching.enableCompanding = not StereoConfigHandler.config.costMatching.enableCompanding state = "on" if StereoConfigHandler.config.costMatching.enableCompanding else "off" print(f"Companding {state}") - elif key == ord('v'): + elif key == ord("v"): StereoConfigHandler.newConfig = True StereoConfigHandler.config.censusTransform.enableMeanMode = not StereoConfigHandler.config.censusTransform.enableMeanMode state = "on" if StereoConfigHandler.config.censusTransform.enableMeanMode else "off" print(f"Census transform mean mode {state}") - elif key == ord('1'): + elif key == ord("1"): StereoConfigHandler.newConfig = True StereoConfigHandler.config.algorithmControl.enableLeftRightCheck = not StereoConfigHandler.config.algorithmControl.enableLeftRightCheck state = "on" if StereoConfigHandler.config.algorithmControl.enableLeftRightCheck else "off" print(f"LR-check {state}") - elif key == ord('2'): + elif key == ord("2"): StereoConfigHandler.newConfig = True StereoConfigHandler.config.algorithmControl.enableSubpixel = not StereoConfigHandler.config.algorithmControl.enableSubpixel state = "on" if StereoConfigHandler.config.algorithmControl.enableSubpixel else "off" print(f"Subpixel {state}") - elif key == ord('3'): + elif key == ord("3"): StereoConfigHandler.newConfig = True StereoConfigHandler.config.algorithmControl.enableExtended = not StereoConfigHandler.config.algorithmControl.enableExtended state = "on" if StereoConfigHandler.config.algorithmControl.enableExtended else "off" @@ -430,28 +473,28 @@ def updateDefaultConfig(config): def registerWindow(stream): cv2.namedWindow(stream, cv2.WINDOW_NORMAL) - StereoConfigHandler.trConfidence.append(StereoConfigHandler.Trackbar('Disparity confidence', stream, 0, 255, StereoConfigHandler.config.costMatching.confidenceThreshold, StereoConfigHandler.trackbarConfidence)) - StereoConfigHandler.trSigma.append(StereoConfigHandler.Trackbar('Bilateral sigma', stream, 0, 100, StereoConfigHandler.config.postProcessing.bilateralSigmaValue, StereoConfigHandler.trackbarSigma)) - StereoConfigHandler.trLrCheck.append(StereoConfigHandler.Trackbar('LR-check threshold', stream, 0, 16, StereoConfigHandler.config.algorithmControl.leftRightCheckThreshold, StereoConfigHandler.trackbarLrCheckThreshold)) - StereoConfigHandler.trFractionalBits.append(StereoConfigHandler.Trackbar('Subpixel fractional bits', stream, 3, 5, StereoConfigHandler.config.algorithmControl.subpixelFractionalBits, StereoConfigHandler.trackbarFractionalBits)) - StereoConfigHandler.trDisparityShift.append(StereoConfigHandler.Trackbar('Disparity shift', stream, 0, 100, StereoConfigHandler.config.algorithmControl.disparityShift, StereoConfigHandler.trackbarDisparityShift)) - StereoConfigHandler.trCenterAlignmentShift.append(StereoConfigHandler.Trackbar('Center alignment shift factor', stream, 0, 100, StereoConfigHandler.config.algorithmControl.centerAlignmentShiftFactor, StereoConfigHandler.trackbarCenterAlignmentShift)) - StereoConfigHandler.trInvalidateEdgePixels.append(StereoConfigHandler.Trackbar('Invalidate edge pixels', stream, 0, 100, StereoConfigHandler.config.algorithmControl.numInvalidateEdgePixels, StereoConfigHandler.trackbarInvalidateEdgePixels)) - StereoConfigHandler.trLineqAlpha.append(StereoConfigHandler.Trackbar('Linear equation alpha', stream, 0, 15, StereoConfigHandler.config.costMatching.linearEquationParameters.alpha, StereoConfigHandler.trackbarLineqAlpha)) - StereoConfigHandler.trLineqBeta.append(StereoConfigHandler.Trackbar('Linear equation beta', stream, 0, 15, StereoConfigHandler.config.costMatching.linearEquationParameters.beta, StereoConfigHandler.trackbarLineqBeta)) - StereoConfigHandler.trLineqThreshold.append(StereoConfigHandler.Trackbar('Linear equation threshold', stream, 0, 255, StereoConfigHandler.config.costMatching.linearEquationParameters.threshold, StereoConfigHandler.trackbarLineqThreshold)) - StereoConfigHandler.trCostAggregationP1.append(StereoConfigHandler.Trackbar('Cost aggregation P1', stream, 0, 500, StereoConfigHandler.config.costAggregation.horizontalPenaltyCostP1, StereoConfigHandler.trackbarCostAggregationP1)) - StereoConfigHandler.trCostAggregationP2.append(StereoConfigHandler.Trackbar('Cost aggregation P2', stream, 0, 500, StereoConfigHandler.config.costAggregation.horizontalPenaltyCostP2, StereoConfigHandler.trackbarCostAggregationP2)) - StereoConfigHandler.trTemporalAlpha.append(StereoConfigHandler.Trackbar('Temporal filter alpha', stream, 0, 100, int(StereoConfigHandler.config.postProcessing.temporalFilter.alpha*100), StereoConfigHandler.trackbarTemporalFilterAlpha)) - StereoConfigHandler.trTemporalDelta.append(StereoConfigHandler.Trackbar('Temporal filter delta', stream, 0, 100, StereoConfigHandler.config.postProcessing.temporalFilter.delta, StereoConfigHandler.trackbarTemporalFilterDelta)) - StereoConfigHandler.trSpatialAlpha.append(StereoConfigHandler.Trackbar('Spatial filter alpha', stream, 0, 100, int(StereoConfigHandler.config.postProcessing.spatialFilter.alpha*100), StereoConfigHandler.trackbarSpatialFilterAlpha)) - StereoConfigHandler.trSpatialDelta.append(StereoConfigHandler.Trackbar('Spatial filter delta', stream, 0, 100, StereoConfigHandler.config.postProcessing.spatialFilter.delta, StereoConfigHandler.trackbarSpatialFilterDelta)) - StereoConfigHandler.trSpatialHoleFilling.append(StereoConfigHandler.Trackbar('Spatial filter hole filling radius', stream, 0, 16, StereoConfigHandler.config.postProcessing.spatialFilter.holeFillingRadius, StereoConfigHandler.trackbarSpatialFilterHoleFillingRadius)) - StereoConfigHandler.trSpatialNumIterations.append(StereoConfigHandler.Trackbar('Spatial filter number of iterations', stream, 0, 4, StereoConfigHandler.config.postProcessing.spatialFilter.numIterations, StereoConfigHandler.trackbarSpatialFilterNumIterations)) - StereoConfigHandler.trThresholdMinRange.append(StereoConfigHandler.Trackbar('Threshold filter min range', stream, 0, 65, StereoConfigHandler.config.postProcessing.thresholdFilter.minRange, StereoConfigHandler.trackbarThresholdMinRange)) - StereoConfigHandler.trThresholdMaxRange.append(StereoConfigHandler.Trackbar('Threshold filter max range', stream, 0, 65, StereoConfigHandler.config.postProcessing.thresholdFilter.maxRange, StereoConfigHandler.trackbarThresholdMaxRange)) - StereoConfigHandler.trSpeckleRange.append(StereoConfigHandler.Trackbar('Speckle filter range', stream, 0, 240, StereoConfigHandler.config.postProcessing.speckleFilter.speckleRange, StereoConfigHandler.trackbarSpeckleRange)) - StereoConfigHandler.trDecimationFactor.append(StereoConfigHandler.Trackbar('Decimation factor', stream, 1, 4, StereoConfigHandler.config.postProcessing.decimationFilter.decimationFactor, StereoConfigHandler.trackbarDecimationFactor)) + StereoConfigHandler.trConfidence.append(StereoConfigHandler.Trackbar("Disparity confidence", stream, 0, 255, StereoConfigHandler.config.costMatching.confidenceThreshold, StereoConfigHandler.trackbarConfidence)) + StereoConfigHandler.trSigma.append(StereoConfigHandler.Trackbar("Bilateral sigma", stream, 0, 100, StereoConfigHandler.config.postProcessing.bilateralSigmaValue, StereoConfigHandler.trackbarSigma)) + StereoConfigHandler.trLrCheck.append(StereoConfigHandler.Trackbar("LR-check threshold", stream, 0, 16, StereoConfigHandler.config.algorithmControl.leftRightCheckThreshold, StereoConfigHandler.trackbarLrCheckThreshold)) + StereoConfigHandler.trFractionalBits.append(StereoConfigHandler.Trackbar("Subpixel fractional bits", stream, 3, 5, StereoConfigHandler.config.algorithmControl.subpixelFractionalBits, StereoConfigHandler.trackbarFractionalBits)) + StereoConfigHandler.trDisparityShift.append(StereoConfigHandler.Trackbar("Disparity shift", stream, 0, 100, StereoConfigHandler.config.algorithmControl.disparityShift, StereoConfigHandler.trackbarDisparityShift)) + StereoConfigHandler.trCenterAlignmentShift.append(StereoConfigHandler.Trackbar("Center alignment shift factor", stream, 0, 100, StereoConfigHandler.config.algorithmControl.centerAlignmentShiftFactor, StereoConfigHandler.trackbarCenterAlignmentShift)) + StereoConfigHandler.trInvalidateEdgePixels.append(StereoConfigHandler.Trackbar("Invalidate edge pixels", stream, 0, 100, StereoConfigHandler.config.algorithmControl.numInvalidateEdgePixels, StereoConfigHandler.trackbarInvalidateEdgePixels)) + StereoConfigHandler.trLineqAlpha.append(StereoConfigHandler.Trackbar("Linear equation alpha", stream, 0, 15, StereoConfigHandler.config.costMatching.linearEquationParameters.alpha, StereoConfigHandler.trackbarLineqAlpha)) + StereoConfigHandler.trLineqBeta.append(StereoConfigHandler.Trackbar("Linear equation beta", stream, 0, 15, StereoConfigHandler.config.costMatching.linearEquationParameters.beta, StereoConfigHandler.trackbarLineqBeta)) + StereoConfigHandler.trLineqThreshold.append(StereoConfigHandler.Trackbar("Linear equation threshold", stream, 0, 255, StereoConfigHandler.config.costMatching.linearEquationParameters.threshold, StereoConfigHandler.trackbarLineqThreshold)) + StereoConfigHandler.trCostAggregationP1.append(StereoConfigHandler.Trackbar("Cost aggregation P1", stream, 0, 500, StereoConfigHandler.config.costAggregation.horizontalPenaltyCostP1, StereoConfigHandler.trackbarCostAggregationP1)) + StereoConfigHandler.trCostAggregationP2.append(StereoConfigHandler.Trackbar("Cost aggregation P2", stream, 0, 500, StereoConfigHandler.config.costAggregation.horizontalPenaltyCostP2, StereoConfigHandler.trackbarCostAggregationP2)) + StereoConfigHandler.trTemporalAlpha.append(StereoConfigHandler.Trackbar("Temporal filter alpha", stream, 0, 100, int(StereoConfigHandler.config.postProcessing.temporalFilter.alpha*100), StereoConfigHandler.trackbarTemporalFilterAlpha)) + StereoConfigHandler.trTemporalDelta.append(StereoConfigHandler.Trackbar("Temporal filter delta", stream, 0, 100, StereoConfigHandler.config.postProcessing.temporalFilter.delta, StereoConfigHandler.trackbarTemporalFilterDelta)) + StereoConfigHandler.trSpatialAlpha.append(StereoConfigHandler.Trackbar("Spatial filter alpha", stream, 0, 100, int(StereoConfigHandler.config.postProcessing.spatialFilter.alpha*100), StereoConfigHandler.trackbarSpatialFilterAlpha)) + StereoConfigHandler.trSpatialDelta.append(StereoConfigHandler.Trackbar("Spatial filter delta", stream, 0, 100, StereoConfigHandler.config.postProcessing.spatialFilter.delta, StereoConfigHandler.trackbarSpatialFilterDelta)) + StereoConfigHandler.trSpatialHoleFilling.append(StereoConfigHandler.Trackbar("Spatial filter hole filling radius", stream, 0, 16, StereoConfigHandler.config.postProcessing.spatialFilter.holeFillingRadius, StereoConfigHandler.trackbarSpatialFilterHoleFillingRadius)) + StereoConfigHandler.trSpatialNumIterations.append(StereoConfigHandler.Trackbar("Spatial filter number of iterations", stream, 0, 4, StereoConfigHandler.config.postProcessing.spatialFilter.numIterations, StereoConfigHandler.trackbarSpatialFilterNumIterations)) + StereoConfigHandler.trThresholdMinRange.append(StereoConfigHandler.Trackbar("Threshold filter min range", stream, 0, 65, StereoConfigHandler.config.postProcessing.thresholdFilter.minRange, StereoConfigHandler.trackbarThresholdMinRange)) + StereoConfigHandler.trThresholdMaxRange.append(StereoConfigHandler.Trackbar("Threshold filter max range", stream, 0, 65, StereoConfigHandler.config.postProcessing.thresholdFilter.maxRange, StereoConfigHandler.trackbarThresholdMaxRange)) + StereoConfigHandler.trSpeckleRange.append(StereoConfigHandler.Trackbar("Speckle filter range", stream, 0, 240, StereoConfigHandler.config.postProcessing.speckleFilter.speckleRange, StereoConfigHandler.trackbarSpeckleRange)) + StereoConfigHandler.trDecimationFactor.append(StereoConfigHandler.Trackbar("Decimation factor", stream, 1, 4, StereoConfigHandler.config.postProcessing.decimationFilter.decimationFactor, StereoConfigHandler.trackbarDecimationFactor)) def __init__(self, config): print("Control median filter using the 'm' key.") @@ -468,6 +511,8 @@ def __init__(self, config): print("Control left-right check mode using the '1' key.") print("Control subpixel mode using the '2' key.") print("Control extended mode using the '3' key.") + if evaluation_mode: + print("Switch between images using '[' and ']' keys.") StereoConfigHandler.config = config @@ -524,24 +569,24 @@ def __init__(self, config): xoutDebugCostDump = pipeline.create(dai.node.XLinkOut) xinStereoDepthConfig.setStreamName("stereoDepthConfig") -monoLeft.setStreamName('in_left') -monoRight.setStreamName('in_right') - -xoutLeft.setStreamName('left') -xoutRight.setStreamName('right') -xoutDepth.setStreamName('depth') -xoutConfMap.setStreamName('confidence_map') -xoutDisparity.setStreamName('disparity') -xoutRectifLeft.setStreamName('rectified_left') -xoutRectifRight.setStreamName('rectified_right') -xoutStereoCfg.setStreamName('stereo_cfg') +monoLeft.setStreamName("in_left") +monoRight.setStreamName("in_right") + +xoutLeft.setStreamName("left") +xoutRight.setStreamName("right") +xoutDepth.setStreamName("depth") +xoutConfMap.setStreamName("confidence_map") +xoutDisparity.setStreamName("disparity") +xoutRectifLeft.setStreamName("rectified_left") +xoutRectifRight.setStreamName("rectified_right") +xoutStereoCfg.setStreamName("stereo_cfg") if args.debug: - xoutDebugLrCheckIt1.setStreamName('disparity_lr_check_iteration1') - xoutDebugLrCheckIt2.setStreamName('disparity_lr_check_iteration2') - xoutDebugExtLrCheckIt1.setStreamName('disparity_ext_lr_check_iteration1') - xoutDebugExtLrCheckIt2.setStreamName('disparity_ext_lr_check_iteration2') + xoutDebugLrCheckIt1.setStreamName("disparity_lr_check_iteration1") + xoutDebugLrCheckIt2.setStreamName("disparity_lr_check_iteration2") + xoutDebugExtLrCheckIt1.setStreamName("disparity_ext_lr_check_iteration1") + xoutDebugExtLrCheckIt2.setStreamName("disparity_ext_lr_check_iteration2") if args.dumpdisparitycostvalues: - xoutDebugCostDump.setStreamName('disparity_cost_dump') + xoutDebugCostDump.setStreamName("disparity_cost_dump") # Properties stereo.setDefaultProfilePreset(dai.node.StereoDepth.PresetMode.HIGH_DENSITY) @@ -549,6 +594,10 @@ def __init__(self, config): stereo.setLeftRightCheck(lrcheck) stereo.setExtendedDisparity(extended) stereo.setSubpixel(subpixel) + +# Switching depthAlign mode at runtime is not supported while aligning to a specific camera is enabled +# stereo.setDepthAlign(dai.CameraBoardSocket.LEFT) + # allocates resources for worst case scenario # allowing runtime switch of stereo modes stereo.setRuntimeModeSwitch(True) @@ -578,7 +627,7 @@ def __init__(self, config): StereoConfigHandler(stereo.initialConfig.get()) -StereoConfigHandler.registerWindow('Stereo control panel') +StereoConfigHandler.registerWindow("Stereo control panel") # stereo.setPostProcessingHardwareResources(3, 3) @@ -588,20 +637,20 @@ def __init__(self, config): fov = 71.86 focal = width / (2 * math.tan(fov / 2 / 180 * math.pi)) -streams = ['left', 'right'] +streams = ["left", "right"] if outRectified: - streams.extend(['rectified_left', 'rectified_right']) -streams.append('disparity') + streams.extend(["rectified_left", "rectified_right"]) +streams.append("disparity") if outDepth: - streams.append('depth') + streams.append("depth") if outConfidenceMap: - streams.append('confidence_map') + streams.append("confidence_map") debugStreams = [] if args.debug: - debugStreams.extend(['disparity_lr_check_iteration1', 'disparity_lr_check_iteration2']) - debugStreams.extend(['disparity_ext_lr_check_iteration1', 'disparity_ext_lr_check_iteration2']) + debugStreams.extend(["disparity_lr_check_iteration1", "disparity_lr_check_iteration2"]) + debugStreams.extend(["disparity_ext_lr_check_iteration1", "disparity_ext_lr_check_iteration2"]) if args.dumpdisparitycostvalues: - debugStreams.append('disparity_cost_dump') + debugStreams.append("disparity_cost_dump") def convertToCv2Frame(name, image, config): @@ -614,19 +663,19 @@ def convertToCv2Frame(name, image, config): # frame.tofile(name+".raw") - if name == 'depth': + if name == "depth": dispScaleFactor = baseline * focal - with np.errstate(divide='ignore'): + with np.errstate(divide="ignore"): frame = dispScaleFactor / frame frame = (frame * 255. / dispIntegerLevels).astype(np.uint8) frame = cv2.applyColorMap(frame, cv2.COLORMAP_HOT) - elif 'confidence_map' in name: + elif "confidence_map" in name: pass - elif name == 'disparity_cost_dump': - # frame.tofile(name+'.raw') + elif name == "disparity_cost_dump": + # frame.tofile(name+".raw") pass - elif 'disparity' in name: + elif "disparity" in name: if 1: # Optionally, extend disparity range to better visualize it frame = (frame * 255. / maxDisp).astype(np.uint8) return frame @@ -635,12 +684,177 @@ def convertToCv2Frame(name, image, config): return frame +class DatasetManager: + def __init__(self, path): + self.path = path + self.index = 0 + self.names = [d for d in os.listdir(path) if os.path.isdir(os.path.join(path, d))] + if len(self.names) == 0: + raise RuntimeError("No dataset found at {}".format(path)) + + def get(self): + return os.path.join(self.path, self.names[self.index]) + + def get_name(self): + return self.names[self.index] + + def next(self): + self.index = (self.index + 1) % len(self.names) + return self.get() + + def prev(self): + self.index = (self.index - 1) % len(self.names) + return self.get() + + +def read_pfm(file): + file = open(file, "rb") + + color = None + width = None + height = None + scale = None + endian = None + + header = file.readline().rstrip() + if header.decode("ascii") == "PF": + color = True + elif header.decode("ascii") == "Pf": + color = False + else: + raise Exception("Not a PFM file.") + + dim_match = re.search(r"(\d+)\s(\d+)", file.readline().decode("ascii")) + if dim_match: + width, height = map(int, dim_match.groups()) + else: + raise Exception("Malformed PFM header.") + + scale = float(file.readline().rstrip()) + if scale < 0: # little-endian + endian = "<" + scale = -scale + else: + endian = ">" # big-endian + + data = np.fromfile(file, endian + "f") + shape = (height, width, 3) if color else (height, width) + return np.flip(np.reshape(data, shape), axis=0), scale + +def calculate_err_measures(gt_img, oak_img): + assert gt_img.shape == oak_img.shape + + gt_mask = gt_img != np.inf + oak_mask = oak_img != np.inf + mask = gt_mask & oak_mask + + gt_img[~gt_mask] = 0. + oak_img[~mask] = 0. + err = np.abs(gt_img - oak_img) + + n = np.sum(gt_mask) + invalid = np.sum(gt_mask & ~oak_mask) + + bad05 = np.sum(mask & (err > 0.5)) + bad1 = np.sum(mask & (err > 1.)) + bad2 = np.sum(mask & (err > 2.)) + bad4 = np.sum(mask & (err > 4.)) + sum_err = np.sum(err[mask]) + sum_sq_err = np.sum(err[mask] ** 2) + errs = err[mask] + + bad05_p = 100. * bad05 / n + total_bad05_p = 100. * (bad05 + invalid) / n + bad1_p = 100. * bad1 / n + total_bad1_p = 100. * (bad1 + invalid) / n + bad2_p = 100. * bad2 / n + total_bad2_p = 100. * (bad2 + invalid) / n + bad4_p = 100. * bad4 / n + total_bad4_p = 100. * (bad4 + invalid) / n + invalid_p = 100. * invalid / n + avg_err = sum_err / (n - invalid) + mse = sum_sq_err / (n - invalid) + a50 = np.percentile(errs, 50) + a90 = np.percentile(errs, 90) + a95 = np.percentile(errs, 95) + a99 = np.percentile(errs, 99) + + return { + "bad0.5": bad05_p, + "total_bad0.5": total_bad05_p, + "bad1": bad1_p, + "total_bad1": total_bad1_p, + "bad2": bad2_p, + "total_bad2": total_bad2_p, + "bad4": bad4_p, + "total_bad4": total_bad4_p, + "invalid": invalid_p, + "avg_err": avg_err, + "mse": mse, + "a50": a50, + "a90": a90, + "a95": a95, + "a99": a99 + } + +def show_evaluation(img_name, evals): + cv2.namedWindow("Evaluation", cv2.WINDOW_NORMAL) + font = cv2.FONT_HERSHEY_SIMPLEX + font_scale = 2 + thickness = 3 + color = (0, 0, 0) + lines = [ + f"Name: {img_name}", + f"Bad0.5: {evals['bad0.5']:.2f}%", + f"Total Bad0.5: {evals['total_bad0.5']:.2f}%", + f"Bad1: {evals['bad1']:.2f}%", + f"Total Bad1: {evals['total_bad1']:.2f}%", + f"Bad2: {evals['bad2']:.2f}%", + f"Total Bad2: {evals['total_bad2']:.2f}%", + f"Bad4: {evals['bad4']:.2f}%", + f"Total Bad4: {evals['total_bad4']:.2f}%", + f"Invalid: {evals['invalid']:.2f}%", + f"Avg Err: {evals['avg_err']:.2f}", + f"MSE: {evals['mse']:.2f}", + f"A50: {evals['a50']:.2f}", + f"A90: {evals['a90']:.2f}", + f"A95: {evals['a95']:.2f}", + f"A99: {evals['a99']:.2f}" + ] + sizes = [cv2.getTextSize(line, font, font_scale, thickness) for line in lines] + sizes = [(size[0][0], size[0][1] + size[1], size[1]) for size in sizes] + max_width = max([size[0] for size in sizes]) + total_height = sum([size[1] for size in sizes]) + (len(lines) - 1) * thickness + img = np.ones((total_height + thickness, max_width, 3), dtype=np.uint8) * 255 + y = 0 + for line, size in zip(lines, sizes): + cv2.putText(img, line, (0, y + size[1] - size[2]), font, font_scale, color, thickness) + y += size[1] + thickness + cv2.imshow("Evaluation", img) + +def show_debug_disparity(gt_img, oak_img): + def rescale_img(img): + img[img == np.inf] = 0. + img = cv2.resize(img, (1280, 800), interpolation=cv2.INTER_AREA) + return img.astype(np.uint16) + + gt_img = rescale_img(gt_img) + oak_img = rescale_img(oak_img) + maxv = max(gt_img.max(), oak_img.max()) + gt_img = (gt_img * 255. / maxv).astype(np.uint8) + oak_img = (oak_img * 255. / maxv).astype(np.uint8) + cv2.imshow("GT", gt_img) + cv2.imshow("OAK", oak_img) + +if evaluation_mode: + dataset = DatasetManager(args.evaluate) + print("Connecting and starting the pipeline") # Connect to device and start pipeline with dai.Device(pipeline) as device: stereoDepthConfigInQueue = device.getInputQueue("stereoDepthConfig") - inStreams = ['in_left', 'in_right'] + inStreams = ["in_left", "in_right"] inStreamsCameraID = [dai.CameraBoardSocket.CAM_B, dai.CameraBoardSocket.CAM_C] in_q_list = [] for s in inStreams: @@ -653,7 +867,6 @@ def convertToCv2Frame(name, image, config): q = device.getOutputQueue(s, 8, blocking=False) q_list.append(q) - inCfg = device.getOutputQueue("stereo_cfg", 8, blocking=False) # Need to set a timestamp for input frames, for the sync stage in Stereo node @@ -666,7 +879,7 @@ def convertToCv2Frame(name, image, config): dataset_size = 1 # Number of image pairs frame_interval_ms = 50 for i, q in enumerate(in_q_list): - path = args.dataset + '/' + str(index) + '/' + q.getName() + '.png' + path = os.path.join(dataset.get(), f"im{i}.png") if evaluation_mode else args.dataset + "/" + str(index) + "/" + q.getName() + ".png" data = cv2.imread(path, cv2.IMREAD_GRAYSCALE) data = cv2.resize(data, (width, height), interpolation = cv2.INTER_AREA) data = data.reshape(height*width) @@ -680,11 +893,16 @@ def convertToCv2Frame(name, image, config): img.setWidth(width) img.setHeight(height) q.send(img) - # print("Sent frame: {:25s}".format(path), 'timestamp_ms:', timestamp_ms) + # print("Sent frame: {:25s}".format(path), "timestamp_ms:", timestamp_ms) timestamp_ms += frame_interval_ms index = (index + 1) % dataset_size sleep(frame_interval_ms / 1000) + gt_disparity = None + if evaluation_mode: + # Load GT disparity + gt_disparity = read_pfm(os.path.join(dataset.get(), f"disp1.pfm"))[0] + # Handle output streams currentConfig = inCfg.get() @@ -693,7 +911,7 @@ def convertToCv2Frame(name, image, config): queues = q_list.copy() if args.dumpdisparitycostvalues: - q = device.getOutputQueue('disparity_cost_dump', 8, blocking=False) + q = device.getOutputQueue("disparity_cost_dump", 8, blocking=False) queues.append(q) if args.debug: @@ -701,11 +919,11 @@ def convertToCv2Frame(name, image, config): activeDebugStreams = [] if lrCheckEnabled: - activeDebugStreams.extend(['disparity_lr_check_iteration1', 'disparity_lr_check_iteration2']) + activeDebugStreams.extend(["disparity_lr_check_iteration1", "disparity_lr_check_iteration2"]) if extendedEnabled: - activeDebugStreams.extend(['disparity_ext_lr_check_iteration1']) + activeDebugStreams.extend(["disparity_ext_lr_check_iteration1"]) if lrCheckEnabled: - activeDebugStreams.extend(['disparity_ext_lr_check_iteration2']) + activeDebugStreams.extend(["disparity_ext_lr_check_iteration2"]) for s in activeDebugStreams: q = device.getOutputQueue(s, 8, blocking=False) @@ -722,14 +940,37 @@ def ListDiff(li1, li2): cv2.destroyWindow(name) prevQueues = queues.copy() + disparity = None for q in queues: - if q.getName() in ['left', 'right']: continue + if q.getName() in ["left", "right"]: continue data = q.get() + if q.getName() == "disparity": + disparity = data.getFrame() frame = convertToCv2Frame(q.getName(), data, currentConfig) cv2.imshow(q.getName(), frame) + if disparity is not None and gt_disparity is not None: + subpixel_bits = 1 << currentConfig.get().algorithmControl.subpixelFractionalBits + subpixel_enabled = currentConfig.get().algorithmControl.enableSubpixel + width_scale = float(gt_disparity.shape[1]) / float(disparity.shape[1]) + + disparity = disparity.astype(np.float32) + if subpixel_enabled: + disparity = disparity / subpixel_bits + disparity = disparity * width_scale + disparity = cv2.resize(disparity, (gt_disparity.shape[1], gt_disparity.shape[0]), interpolation = cv2.INTER_LINEAR) + disparity[disparity == 0.] = np.inf + + # show_debug_disparity(gt_disparity, disparity) + err_vals = calculate_err_measures(gt_disparity, disparity) + show_evaluation(dataset.get_name(), err_vals) + key = cv2.waitKey(1) - if key == ord('q'): + if key == ord("q"): break + elif evaluation_mode and key == ord("["): + dataset.next() + elif evaluation_mode and key == ord("]"): + dataset.prev() StereoConfigHandler.handleKeypress(key, stereoDepthConfigInQueue) From c971026a591d991e228255e101f11052cd48f259 Mon Sep 17 00:00:00 2001 From: Erol444 Date: Fri, 11 Aug 2023 18:56:21 +0300 Subject: [PATCH 42/87] Updated flash_bootlaoder example, so it will try to flash user bootloader (if its available) --- examples/bootloader/flash_bootloader.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/examples/bootloader/flash_bootloader.py b/examples/bootloader/flash_bootloader.py index 50fc077da..b57234161 100755 --- a/examples/bootloader/flash_bootloader.py +++ b/examples/bootloader/flash_bootloader.py @@ -56,10 +56,13 @@ # Create a progress callback lambda progress = lambda p : print(f'Flashing progress: {p*100:.1f}%') - - print(f"Flashing {blType.name} bootloader...") startTime = time.monotonic() - (res, message) = bl.flashBootloader(dai.DeviceBootloader.Memory.FLASH, blType, progress) + is_user_bl = bl.isUserBootloaderSupported() + print(f"Flashing {blType.name} {'USER' if is_user_bl else 'FACTORY'} bootloader...") + if is_user_bl: + (res, message) = bl.flashUserBootloader(progress) + else: # Factory bootloader + (res, message) = bl.flashBootloader(dai.DeviceBootloader.Memory.FLASH, blType, progress) if res: print("Flashing successful. Took", time.monotonic() - startTime, "seconds") else: From 76e52342b323c1e0d921fa31cc08de2ac3fdb072 Mon Sep 17 00:00:00 2001 From: asahtik Date: Mon, 14 Aug 2023 16:05:26 +0200 Subject: [PATCH 43/87] Added getter and setter bindings --- src/pipeline/datatype/BufferBindings.cpp | 27 ++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/pipeline/datatype/BufferBindings.cpp b/src/pipeline/datatype/BufferBindings.cpp index df1a2ea7c..34f8137f2 100644 --- a/src/pipeline/datatype/BufferBindings.cpp +++ b/src/pipeline/datatype/BufferBindings.cpp @@ -42,6 +42,27 @@ void bind_buffer(pybind11::module& m, void* pCallstack){ dai::RawBuffer &a = obj.cast(); a.data = {array.data(), array.data() + array.size()}; }) + .def_property("ts", + [](const RawBuffer& o){ + double ts = o.ts.sec + o.ts.nsec / 1000000000.0; + return ts; + }, + [](RawBuffer& o, double ts){ + o.ts.sec = ts; + o.ts.nsec = (ts - o.ts.sec) * 1000000000.0; + } + ) + .def_property("tsDevice", + [](const RawBuffer& o){ + double ts = o.tsDevice.sec + o.tsDevice.nsec / 1000000000.0; + return ts; + }, + [](RawBuffer& o, double ts){ + o.tsDevice.sec = ts; + o.tsDevice.nsec = (ts - o.tsDevice.sec) * 1000000000.0; + } + ) + .def_readwrite("sequenceNum", &RawBuffer::sequenceNum) ; // Message @@ -59,6 +80,12 @@ void bind_buffer(pybind11::module& m, void* pCallstack){ buffer.getData().clear(); buffer.getData().insert(buffer.getData().begin(), array.data(), array.data() + array.nbytes()); }, DOC(dai, Buffer, setData)) + .def("getTimestamp", &Buffer::getTimestamp, DOC(dai, Buffer, getTimestamp)) + .def("getTimestampDevice", &Buffer::getTimestampDevice, DOC(dai, Buffer, getTimestampDevice)) + .def("getSequenceNum", &Buffer::getSequenceNum, DOC(dai, Buffer, getSequenceNum)) + .def("setTimestamp", &Buffer::setTimestamp, DOC(dai, Buffer, setTimestamp)) + .def("setTimestampDevice", &Buffer::setTimestampDevice, DOC(dai, Buffer, setTimestampDevice)) + .def("setSequenceNum", &Buffer::setSequenceNum, DOC(dai, Buffer, setSequenceNum)) ; From 529382ba103d30563a49eb9a78f4f68ed3699cf3 Mon Sep 17 00:00:00 2001 From: Erol444 Date: Wed, 16 Aug 2023 11:31:29 +0200 Subject: [PATCH 44/87] Updated readthedocs.yaml, as they decided to deprecate some functions yet again for absolutely no good reason --- .readthedocs.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 14ba0d775..4358f76f5 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -6,7 +6,9 @@ version: 2 build: - image: latest + os: ubuntu-20.04 + tools: + python: "3.8" # Submodules submodules: @@ -29,6 +31,5 @@ formats: [] # Optionally set the version of Python and requirements required to build your docs python: - version: 3.8 install: - requirements: docs/readthedocs/requirements.txt \ No newline at end of file From dd4920340ae8ee0cc9c10a0213b47187fec4daeb Mon Sep 17 00:00:00 2001 From: alex-luxonis Date: Thu, 17 Aug 2023 16:39:43 +0300 Subject: [PATCH 45/87] FW: add support for OAK-D-SR-PoE R1M1E1 --- depthai-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/depthai-core b/depthai-core index 0e37dcde9..5198e8b6e 160000 --- a/depthai-core +++ b/depthai-core @@ -1 +1 @@ -Subproject commit 0e37dcde91afc4c633da40eb47887ec266314733 +Subproject commit 5198e8b6e177e5be00e16ae3a6d0e9090267ee20 From efcbd7d48b5c55408154c9bc68bac38e5c25e551 Mon Sep 17 00:00:00 2001 From: Martin Peterlin Date: Mon, 21 Aug 2023 09:30:03 +0200 Subject: [PATCH 46/87] Modified default developement version from dev to dev0 --- find_version.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/find_version.py b/find_version.py index 057794131..610bd126e 100644 --- a/find_version.py +++ b/find_version.py @@ -12,7 +12,7 @@ def get_version_from_cmake_lists(path): with open(path, 'r') as file: content = file.read() - match = re.search(cmake_lists_txt_version_pattern, content, flags=re.IGNORECASE) + match = re.search(cmake_lists_txt_version_pattern, content, flags=re.IGNORECASE) ver1 = match.group('ver1') ver2 = match.group('ver2') version = ver1 @@ -21,7 +21,7 @@ def get_version_from_cmake_lists(path): return version def get_package_version(): - + version_core = '0.0.0' version_revision = '0' version_core = get_version_from_cmake_lists(version_depthai_core_path) @@ -32,4 +32,4 @@ def get_package_version(): def get_package_dev_version(commit_hash): - return get_package_version() + ".dev+" + commit_hash \ No newline at end of file + return get_package_version() + ".dev0+" + commit_hash \ No newline at end of file From 25c7699b2133a0deb3a463295b2658f58f734ae8 Mon Sep 17 00:00:00 2001 From: Martin Peterlin Date: Mon, 21 Aug 2023 09:30:30 +0200 Subject: [PATCH 47/87] Improved dll library copying --- CMakeLists.txt | 3 ++- depthai-core | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4bdd118e6..d8fc4fee8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -162,8 +162,9 @@ if(WIN32) set(depthai_dll_libraries "$") endif() add_custom_command(TARGET ${TARGET_NAME} POST_BUILD COMMAND - ${CMAKE_COMMAND} -E copy ${depthai_dll_libraries} $ + "$<$:${CMAKE_COMMAND};-E;copy_if_different;${depthai_dll_libraries};$>" COMMAND_EXPAND_LISTS + VERBATIM ) # Disable "d" postfix, so python can import the library as is diff --git a/depthai-core b/depthai-core index 5198e8b6e..4310b9ded 160000 --- a/depthai-core +++ b/depthai-core @@ -1 +1 @@ -Subproject commit 5198e8b6e177e5be00e16ae3a6d0e9090267ee20 +Subproject commit 4310b9ded5ac2e09b69ab853a982646d0cb0df99 From 3e7e9e9ed56b1be74f5b96704ff910096d6af55e Mon Sep 17 00:00:00 2001 From: Martin Peterlin Date: Mon, 21 Aug 2023 10:04:44 +0200 Subject: [PATCH 48/87] Updated XLink library with some fixes --- depthai-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/depthai-core b/depthai-core index 4310b9ded..5e653002a 160000 --- a/depthai-core +++ b/depthai-core @@ -1 +1 @@ -Subproject commit 4310b9ded5ac2e09b69ab853a982646d0cb0df99 +Subproject commit 5e653002a4982428518ee12f98ea565e186e2386 From 1b1b3a6c59ceb15ce3942e702ab5b60899c0c2c6 Mon Sep 17 00:00:00 2001 From: alex-luxonis Date: Wed, 23 Aug 2023 11:39:50 -0400 Subject: [PATCH 49/87] FW: fix default fsync on OAK-D-SR-PoE. GPIO46 input by default --- depthai-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/depthai-core b/depthai-core index 5e653002a..26cc2fb52 160000 --- a/depthai-core +++ b/depthai-core @@ -1 +1 @@ -Subproject commit 5e653002a4982428518ee12f98ea565e186e2386 +Subproject commit 26cc2fb5282afc3a467f39aa0ec709f5d22a11ed From d5c9824aba6591c7d9a74425c558d4a4b85e7f10 Mon Sep 17 00:00:00 2001 From: Erol444 Date: Wed, 23 Aug 2023 20:29:56 -0400 Subject: [PATCH 50/87] Updated bootloader docs/examples --- docs/source/components/bootloader.rst | 14 ++++- .../samples/bootloader/bootloader_version.rst | 6 +- .../samples/bootloader/flash_bootloader.rst | 24 +++++--- .../bootloader/flash_user_bootloader.rst | 56 +++++++++++++++++++ docs/source/tutorials/code_samples.rst | 3 +- examples/bootloader/flash_bootloader.py | 21 ++++--- 6 files changed, 104 insertions(+), 20 deletions(-) create mode 100644 docs/source/samples/bootloader/flash_user_bootloader.rst diff --git a/docs/source/components/bootloader.rst b/docs/source/components/bootloader.rst index d9660e713..fc678f56b 100644 --- a/docs/source/components/bootloader.rst +++ b/docs/source/components/bootloader.rst @@ -4,9 +4,9 @@ Bootloader ========== DepthAI bootloader is a small program which handles the booting process, either by **booting the flashed application** (see :ref:`Standalone mode`), -or by **initializing the OAK PoE camera** so DepthAI API can connect to it. OAK PoE cameras already come with bootloader flashed at the factory. +or by **initializing the OAK PoE camera** so DepthAI API can connect to it. OAK PoE cameras already come with factory bootloader flashed at the factory. -Bootloader is part of the ``depthai`` library, so to eg. flash the newest bootloader, you should use the newest ``depthai`` library. +Bootloader is bundled inside with the ``depthai`` library, so to eg. flash the newest bootloader, you should use the newest ``depthai`` library. Device Manager ############## @@ -81,6 +81,16 @@ Device Manager will try to flash the user bootloader first, if flashed (factory) * **Factory reset** will erase the whole flash content and re-flash it with only the USB or NETWORK bootloader. Flashed application (pipeline, assets) and bootloader configurations will be lost. * **Boot into USB recovery mode** will force eg. OAK PoE camera to be available through the USB connector, even if its boot pins are set to PoE booting. It is mostly used by our firmware developers. +Factory and User bootloader +########################### + +There are two types of bootloaders: + +- **Factory bootloader**: bootloader that is flashed in the factory. We don't recommend re-flashing this bootloader, as it is not meant to be edited by end users. +- **User bootloader**: bootloader that can be flashed by the user. If booting is unsuccessful (eg. gets corrupted when flashing), it will fallback to factory bootloader. + +USB devices don't support user bootloader. + Boot switches ############# diff --git a/docs/source/samples/bootloader/bootloader_version.rst b/docs/source/samples/bootloader/bootloader_version.rst index 7b2479a78..77d00c926 100644 --- a/docs/source/samples/bootloader/bootloader_version.rst +++ b/docs/source/samples/bootloader/bootloader_version.rst @@ -16,8 +16,10 @@ Example script output .. code-block:: bash ~/depthai-python/examples$ python3 bootloader_version.py - Found device with name: 14442C10D1789ACD00-ma2480 - Version: 0.0.15 + Found device with name: 1.1 + Version: 0.0.26 + USB Bootloader - supports only Flash memory + Memory 'Memory.FLASH' size: 33554432, info: JEDEC ID: 01 02 19 Setup ##### diff --git a/docs/source/samples/bootloader/flash_bootloader.rst b/docs/source/samples/bootloader/flash_bootloader.rst index 4b4b9e32e..c52891777 100644 --- a/docs/source/samples/bootloader/flash_bootloader.rst +++ b/docs/source/samples/bootloader/flash_bootloader.rst @@ -1,10 +1,9 @@ Flash Bootloader ================ -This script will flash bootloader to the connected OAK camera. Bootloader can only be flashed to devices that -have flash on-board. - -Click on :ref:`Bootloader` for more information. +This script will flash :ref:`factory bootloader ` to the connected OAK camera. It will +also clear the :ref:`user bootloader ` (if it exists). +:ref:`Bootloader` can only be flashed to devices that have on-board flash memory. .. note:: We suggest using :ref:`Device Manager`, a GUI tool for interfacing with the bootloader and its configurations. @@ -16,9 +15,20 @@ Example script output .. code-block:: bash - ~/depthai-python/examples$ python3 bootloader_version.py - Found device with name: 14442C1031425FD700-ma2480 - Version: 0.0.15 + ~/depthai-python/examples$ python3 flash_bootloader.py + [0] 1844301041C83D0E00 [X_LINK_USB_VSC] + Which DepthAI device to flash bootloader for [0..0]: 0 + Booting latest bootloader first, will take a tad longer... + Bootloader version to flash: 0.0.26 + No config found, skipping erasing user bootloader + Flashing USB bootloader... + Flashing progress: 0.0% + Flashing progress: 18.8% + Flashing progress: 31.2% + Flashing progress: 48.2% + Flashing progress: 94.2% + Flashing progress: 100.0% + Flashing successful. Took 7.55600329185836 seconds Setup ##### diff --git a/docs/source/samples/bootloader/flash_user_bootloader.rst b/docs/source/samples/bootloader/flash_user_bootloader.rst new file mode 100644 index 000000000..1a2ac421e --- /dev/null +++ b/docs/source/samples/bootloader/flash_user_bootloader.rst @@ -0,0 +1,56 @@ +Flash User Bootloader +===================== + +This script will flash :ref:`user bootloader ` to the connected OAK camera. +:ref:`Bootloader` can only be flashed to devices that have on-board flash memory. + +.. note:: + We suggest using :ref:`Device Manager`, a GUI tool for interfacing with the bootloader and its configurations. + +Demo +#### + +Example script output + +.. code-block:: bash + + ~/depthai-python/examples/bootloader$ python3 flash_user_bootloader.py + [0] 1844301041C83D0E00 [X_LINK_USB_VSC] current bootloader: 0.0.26 + Which DepthAI device to flash User Bootloader for (Note: Only NETWORK supported) [0..0]: 0 + User Bootloader version to flash: 0.0.26 + Flashing User Bootloader... + Flashing progress: 0.0% + Flashing progress: 18.8% + Flashing progress: 31.2% + Flashing progress: 48.2% + Flashing progress: 94.2% + Flashing progress: 100.0% + Flashing successful. Took 7.55600329185836 seconds + +Setup +##### + +.. include:: /includes/install_from_pypi.rst + +Source code +########### + +.. tabs:: + + .. tab:: Python + + Also `available on GitHub `__ + + .. literalinclude:: ../../../../examples/bootloader/flash_user_bootloader.py + :language: python + :linenos: + + .. tab:: C++ + + Also `available on GitHub `__ + + .. literalinclude:: ../../../../depthai-core/examples/bootloader/flash_user_bootloader.cpp + :language: cpp + :linenos: + +.. include:: /includes/footer-short.rst diff --git a/docs/source/tutorials/code_samples.rst b/docs/source/tutorials/code_samples.rst index 193a04370..4b96ba2a0 100644 --- a/docs/source/tutorials/code_samples.rst +++ b/docs/source/tutorials/code_samples.rst @@ -35,7 +35,8 @@ are presented with code. - :ref:`Bootloader Config` - Read/Flash/Clear bootloader on the device - :ref:`Bootloader Version` - Retrieves Version of Bootloader on the device -- :ref:`Flash Bootloader` - Flash the bootloader to the device +- :ref:`Flash Bootloader` - Flash factory bootloader to the device +- :ref:`Flash User Bootloader` - Flash user bootloader to the device - :ref:`POE set IP` - Sets IP of the POE device .. rubric:: Calibration diff --git a/examples/bootloader/flash_bootloader.py b/examples/bootloader/flash_bootloader.py index b57234161..aca054781 100755 --- a/examples/bootloader/flash_bootloader.py +++ b/examples/bootloader/flash_bootloader.py @@ -25,12 +25,12 @@ with dai.DeviceBootloader(di) as bl: print(f' current bootloader: {bl.getVersion()}', end='') print() - selected = input(f'Which DepthAI device to flash bootloader for [0..{len(deviceInfos)-1}]: ') + selected = input(f'Which OAK device to flash factory bootloader for [0..{len(deviceInfos)-1}]: ') info = deviceInfos[int(selected)] hasBootloader = (info.state == dai.XLinkDeviceState.X_LINK_BOOTLOADER) if hasBootloader: - print("Warning! Flashing bootloader can potentially soft brick your device and should be done with caution.") + print("Warning! Flashing factory bootloader can potentially soft brick your device and should be done with caution.") print("Do not unplug your device while the bootloader is flashing.") print("Type 'y' and press enter to proceed, otherwise exits: ") if input() != 'y': @@ -54,15 +54,20 @@ print("Prompt declined, exiting...") exit(-1) + try: + # Clears out user bootloader + configJson = bl.readConfigData() + configJson["userBlSize"] = 0 + bl.flashConfigData(configJson) + except: + print('No config found, skipping erasing user bootloader') + # Create a progress callback lambda progress = lambda p : print(f'Flashing progress: {p*100:.1f}%') + + print(f"Flashing {blType.name} bootloader...") startTime = time.monotonic() - is_user_bl = bl.isUserBootloaderSupported() - print(f"Flashing {blType.name} {'USER' if is_user_bl else 'FACTORY'} bootloader...") - if is_user_bl: - (res, message) = bl.flashUserBootloader(progress) - else: # Factory bootloader - (res, message) = bl.flashBootloader(dai.DeviceBootloader.Memory.FLASH, blType, progress) + (res, message) = bl.flashBootloader(dai.DeviceBootloader.Memory.FLASH, blType, progress) if res: print("Flashing successful. Took", time.monotonic() - startTime, "seconds") else: From 9fc936aa20d831a0018bc4fe53da0bf6a7f838b4 Mon Sep 17 00:00:00 2001 From: Erol444 Date: Wed, 23 Aug 2023 20:33:20 -0400 Subject: [PATCH 51/87] Added info about bootloader priority in docs --- docs/source/components/bootloader.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/components/bootloader.rst b/docs/source/components/bootloader.rst index fc678f56b..68f4f199f 100644 --- a/docs/source/components/bootloader.rst +++ b/docs/source/components/bootloader.rst @@ -89,7 +89,7 @@ There are two types of bootloaders: - **Factory bootloader**: bootloader that is flashed in the factory. We don't recommend re-flashing this bootloader, as it is not meant to be edited by end users. - **User bootloader**: bootloader that can be flashed by the user. If booting is unsuccessful (eg. gets corrupted when flashing), it will fallback to factory bootloader. -USB devices don't support user bootloader. +USB devices don't support user bootloader. If device has User bootloader, it will be used by default. If user bootloader is not flashed, it will fallback to Factory bootloader. Boot switches ############# From a1c01dadd4a9e1f6a193f106cc420d503746e1e8 Mon Sep 17 00:00:00 2001 From: Aniel Alexa Date: Thu, 24 Aug 2023 18:12:35 +0300 Subject: [PATCH 52/87] hasAutofocusIC bindings --- depthai-core | 2 +- src/pipeline/CommonBindings.cpp | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/depthai-core b/depthai-core index 26cc2fb52..517ce9384 160000 --- a/depthai-core +++ b/depthai-core @@ -1 +1 @@ -Subproject commit 26cc2fb5282afc3a467f39aa0ec709f5d22a11ed +Subproject commit 517ce9384e4de25bec81eff31b390726a429d76f diff --git a/src/pipeline/CommonBindings.cpp b/src/pipeline/CommonBindings.cpp index 8d5f0d50d..eaeb0bab0 100644 --- a/src/pipeline/CommonBindings.cpp +++ b/src/pipeline/CommonBindings.cpp @@ -203,6 +203,7 @@ void CommonBindings::bind(pybind11::module& m, void* pCallstack){ .def_readwrite("orientation", &CameraFeatures::orientation) .def_readwrite("supportedTypes", &CameraFeatures::supportedTypes) .def_readwrite("hasAutofocus", &CameraFeatures::hasAutofocus) + .def_readwrite("hasAutofocusIC", &CameraFeatures::hasAutofocusIC) .def_readwrite("name", &CameraFeatures::name) .def_readwrite("configs", &CameraFeatures::configs) .def("__repr__", [](CameraFeatures& camera) { From a192e088674f72b9194b79f2dc8bd5bac4018d3c Mon Sep 17 00:00:00 2001 From: alex-luxonis Date: Mon, 28 Aug 2023 11:08:02 -0400 Subject: [PATCH 53/87] FW: fix 4 cams crash on PoE due to memory allocation --- depthai-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/depthai-core b/depthai-core index 26cc2fb52..0343e428d 160000 --- a/depthai-core +++ b/depthai-core @@ -1 +1 @@ -Subproject commit 26cc2fb5282afc3a467f39aa0ec709f5d22a11ed +Subproject commit 0343e428d65f0417123b4b6b0dc608a4faf1107b From 675a23b1b9bdc78188bb4f0d0c43d18d8d82ce61 Mon Sep 17 00:00:00 2001 From: SzabolcsGergely Date: Thu, 31 Aug 2023 04:11:04 +0300 Subject: [PATCH 54/87] Update core --- depthai-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/depthai-core b/depthai-core index 0343e428d..cd9d88360 160000 --- a/depthai-core +++ b/depthai-core @@ -1 +1 @@ -Subproject commit 0343e428d65f0417123b4b6b0dc608a4faf1107b +Subproject commit cd9d88360016dee55ddebf6a8d69e3d15bd6f465 From 15a256b619afc418fa3d654fcd0d0083c534489a Mon Sep 17 00:00:00 2001 From: SzabolcsGergely Date: Thu, 31 Aug 2023 04:40:00 +0300 Subject: [PATCH 55/87] Update core --- depthai-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/depthai-core b/depthai-core index cd9d88360..27d49a13d 160000 --- a/depthai-core +++ b/depthai-core @@ -1 +1 @@ -Subproject commit cd9d88360016dee55ddebf6a8d69e3d15bd6f465 +Subproject commit 27d49a13d635f30fc4084c0ffee92844ee20a7c9 From ad04dfa965c5d5b1914b788b857f0940557ad9d0 Mon Sep 17 00:00:00 2001 From: alex-luxonis Date: Thu, 31 Aug 2023 21:15:52 +0300 Subject: [PATCH 56/87] FW: fix OAK-D-SR camera enum. IMX296 updates: - support for RPi GS Cam paired with Luxonis SL6996 RPi camera adapter - support for external trigger on XTR/XTRIG pin: active low, pulse width determines exposure time. Limitation: needs an initial trigger shortly after start to avoid a crash, or `DEPTHAI_WATCHDOG=0` can be a workaround --- depthai-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/depthai-core b/depthai-core index 27d49a13d..47055d527 160000 --- a/depthai-core +++ b/depthai-core @@ -1 +1 @@ -Subproject commit 27d49a13d635f30fc4084c0ffee92844ee20a7c9 +Subproject commit 47055d5270b13adbb740cb1a897b7f9fe7a98aa5 From 496952fa70270ebda8c1a4020df6b49e0b9b4a9e Mon Sep 17 00:00:00 2001 From: SzabolcsGergely Date: Fri, 1 Sep 2023 01:20:10 +0300 Subject: [PATCH 57/87] Fix DOCs --- src/pipeline/datatype/AprilTagsBindings.cpp | 6 +++--- src/pipeline/datatype/ImgDetectionsBindings.cpp | 6 +++--- src/pipeline/datatype/ImgFrameBindings.cpp | 10 +++++----- src/pipeline/datatype/NNDataBindings.cpp | 6 +++--- src/pipeline/datatype/SpatialImgDetectionsBindings.cpp | 6 +++--- .../datatype/SpatialLocationCalculatorDataBindings.cpp | 6 +++--- src/pipeline/datatype/TrackedFeaturesBindings.cpp | 6 +++--- src/pipeline/datatype/TrackletsBindings.cpp | 6 +++--- 8 files changed, 26 insertions(+), 26 deletions(-) diff --git a/src/pipeline/datatype/AprilTagsBindings.cpp b/src/pipeline/datatype/AprilTagsBindings.cpp index 7fb276c5f..66041e92a 100644 --- a/src/pipeline/datatype/AprilTagsBindings.cpp +++ b/src/pipeline/datatype/AprilTagsBindings.cpp @@ -48,9 +48,9 @@ void bind_apriltags(pybind11::module& m, void* pCallstack){ aprilTags .def(py::init<>()) .def_property("aprilTags", [](AprilTags& det) { return &det.aprilTags; }, [](AprilTags& det, std::vector val) { det.aprilTags = val; }) - .def("getTimestamp", &AprilTags::getTimestamp, DOC(dai, AprilTags, getTimestamp)) - .def("getTimestampDevice", &AprilTags::getTimestampDevice, DOC(dai, AprilTags, getTimestampDevice)) - .def("getSequenceNum", &AprilTags::getSequenceNum, DOC(dai, AprilTags, getSequenceNum)) + .def("getTimestamp", &AprilTags::getTimestamp, DOC(dai, Buffer, getTimestamp)) + .def("getTimestampDevice", &AprilTags::getTimestampDevice, DOC(dai, Buffer, getTimestampDevice)) + .def("getSequenceNum", &AprilTags::getSequenceNum, DOC(dai, Buffer, getSequenceNum)) .def("setTimestamp", &AprilTags::setTimestamp, DOC(dai, AprilTags, setTimestamp)) .def("setTimestampDevice", &AprilTags::setTimestampDevice, DOC(dai, AprilTags, setTimestampDevice)) .def("setSequenceNum", &AprilTags::setSequenceNum, DOC(dai, AprilTags, setSequenceNum)) diff --git a/src/pipeline/datatype/ImgDetectionsBindings.cpp b/src/pipeline/datatype/ImgDetectionsBindings.cpp index ef27eea3a..6d8d01b4b 100644 --- a/src/pipeline/datatype/ImgDetectionsBindings.cpp +++ b/src/pipeline/datatype/ImgDetectionsBindings.cpp @@ -74,9 +74,9 @@ void bind_imgdetections(pybind11::module& m, void* pCallstack){ imgDetections .def(py::init<>(), DOC(dai, ImgDetections, ImgDetections)) .def_property("detections", [](ImgDetections& det) { return &det.detections; }, [](ImgDetections& det, std::vector val) { det.detections = val; }, DOC(dai, ImgDetections, detections)) - .def("getTimestamp", &ImgDetections::getTimestamp, DOC(dai, ImgDetections, getTimestamp)) - .def("getTimestampDevice", &ImgDetections::getTimestampDevice, DOC(dai, ImgDetections, getTimestampDevice)) - .def("getSequenceNum", &ImgDetections::getSequenceNum, DOC(dai, ImgDetections, getSequenceNum)) + .def("getTimestamp", &ImgDetections::getTimestamp, DOC(dai, Buffer, getTimestamp)) + .def("getTimestampDevice", &ImgDetections::getTimestampDevice, DOC(dai, Buffer, getTimestampDevice)) + .def("getSequenceNum", &ImgDetections::getSequenceNum, DOC(dai, Buffer, getSequenceNum)) .def("setTimestamp", &ImgDetections::setTimestamp, DOC(dai, ImgDetections, setTimestamp)) .def("setTimestampDevice", &ImgDetections::setTimestampDevice, DOC(dai, ImgDetections, setTimestampDevice)) .def("setSequenceNum", &ImgDetections::setSequenceNum, DOC(dai, ImgDetections, setSequenceNum)) diff --git a/src/pipeline/datatype/ImgFrameBindings.cpp b/src/pipeline/datatype/ImgFrameBindings.cpp index a89ff4230..00b86aba9 100644 --- a/src/pipeline/datatype/ImgFrameBindings.cpp +++ b/src/pipeline/datatype/ImgFrameBindings.cpp @@ -117,13 +117,13 @@ void bind_imgframe(pybind11::module& m, void* pCallstack){ imgFrame .def(py::init<>()) // getters - .def("getTimestamp", py::overload_cast<>(&ImgFrame::getTimestamp, py::const_), DOC(dai, ImgFrame, getTimestamp)) - .def("getTimestampDevice", py::overload_cast<>(&ImgFrame::getTimestampDevice, py::const_), DOC(dai, ImgFrame, getTimestampDevice)) - .def("getTimestamp", py::overload_cast(&ImgFrame::getTimestamp, py::const_), py::arg("offset"), DOC(dai, ImgFrame, getTimestamp)) - .def("getTimestampDevice", py::overload_cast(&ImgFrame::getTimestampDevice, py::const_), py::arg("offset"), DOC(dai, ImgFrame, getTimestampDevice)) + .def("getTimestamp", py::overload_cast<>(&ImgFrame::getTimestamp, py::const_), DOC(dai, Buffer, getTimestamp)) + .def("getTimestampDevice", py::overload_cast<>(&ImgFrame::getTimestampDevice, py::const_), DOC(dai, Buffer, getTimestampDevice)) + .def("getTimestamp", py::overload_cast(&ImgFrame::getTimestamp, py::const_), py::arg("offset"), DOC(dai, Buffer, getTimestamp)) + .def("getTimestampDevice", py::overload_cast(&ImgFrame::getTimestampDevice, py::const_), py::arg("offset"), DOC(dai, Buffer, getTimestampDevice)) .def("getInstanceNum", &ImgFrame::getInstanceNum, DOC(dai, ImgFrame, getInstanceNum)) .def("getCategory", &ImgFrame::getCategory, DOC(dai, ImgFrame, getCategory)) - .def("getSequenceNum", &ImgFrame::getSequenceNum, DOC(dai, ImgFrame, getSequenceNum)) + .def("getSequenceNum", &ImgFrame::getSequenceNum, DOC(dai, Buffer, getSequenceNum)) .def("getWidth", &ImgFrame::getWidth, DOC(dai, ImgFrame, getWidth)) .def("getHeight", &ImgFrame::getHeight, DOC(dai, ImgFrame, getHeight)) .def("getType", &ImgFrame::getType, DOC(dai, ImgFrame, getType)) diff --git a/src/pipeline/datatype/NNDataBindings.cpp b/src/pipeline/datatype/NNDataBindings.cpp index c5b36967a..9d0bef497 100644 --- a/src/pipeline/datatype/NNDataBindings.cpp +++ b/src/pipeline/datatype/NNDataBindings.cpp @@ -125,9 +125,9 @@ void bind_nndata(pybind11::module& m, void* pCallstack){ .def("getFirstLayerUInt8", &NNData::getFirstLayerUInt8, DOC(dai, NNData, getFirstLayerUInt8)) .def("getFirstLayerFp16", &NNData::getFirstLayerFp16, DOC(dai, NNData, getFirstLayerFp16)) .def("getFirstLayerInt32", &NNData::getFirstLayerInt32, DOC(dai, NNData, getFirstLayerInt32)) - .def("getTimestamp", &NNData::getTimestamp, DOC(dai, NNData, getTimestamp)) - .def("getTimestampDevice", &NNData::getTimestampDevice, DOC(dai, NNData, getTimestampDevice)) - .def("getSequenceNum", &NNData::getSequenceNum, DOC(dai, NNData, getSequenceNum)) + .def("getTimestamp", &NNData::getTimestamp, DOC(dai, Buffer, getTimestamp)) + .def("getTimestampDevice", &NNData::getTimestampDevice, DOC(dai, Buffer, getTimestampDevice)) + .def("getSequenceNum", &NNData::getSequenceNum, DOC(dai, Buffer, getSequenceNum)) .def("setTimestamp", &NNData::setTimestamp, DOC(dai, NNData, setTimestamp)) .def("setTimestampDevice", &NNData::setTimestampDevice, DOC(dai, NNData, setTimestampDevice)) .def("setSequenceNum", &NNData::setSequenceNum, DOC(dai, NNData, setSequenceNum)) diff --git a/src/pipeline/datatype/SpatialImgDetectionsBindings.cpp b/src/pipeline/datatype/SpatialImgDetectionsBindings.cpp index a724480c0..86866bc3e 100644 --- a/src/pipeline/datatype/SpatialImgDetectionsBindings.cpp +++ b/src/pipeline/datatype/SpatialImgDetectionsBindings.cpp @@ -70,9 +70,9 @@ void bind_spatialimgdetections(pybind11::module& m, void* pCallstack){ spatialImgDetections .def(py::init<>()) .def_property("detections", [](SpatialImgDetections& det) { return &det.detections; }, [](SpatialImgDetections& det, std::vector val) { det.detections = val; }) - .def("getTimestamp", &SpatialImgDetections::getTimestamp, DOC(dai, SpatialImgDetections, getTimestamp)) - .def("getTimestampDevice", &SpatialImgDetections::getTimestampDevice, DOC(dai, SpatialImgDetections, getTimestampDevice)) - .def("getSequenceNum", &SpatialImgDetections::getSequenceNum, DOC(dai, SpatialImgDetections, getSequenceNum)) + .def("getTimestamp", &SpatialImgDetections::getTimestamp, DOC(dai, Buffer, getTimestamp)) + .def("getTimestampDevice", &SpatialImgDetections::getTimestampDevice, DOC(dai, Buffer, getTimestampDevice)) + .def("getSequenceNum", &SpatialImgDetections::getSequenceNum, DOC(dai, Buffer, getSequenceNum)) .def("setTimestamp", &SpatialImgDetections::setTimestamp, DOC(dai, SpatialImgDetections, setTimestamp)) .def("setTimestampDevice", &SpatialImgDetections::setTimestampDevice, DOC(dai, SpatialImgDetections, setTimestampDevice)) .def("setSequenceNum", &SpatialImgDetections::setSequenceNum, DOC(dai, SpatialImgDetections, setSequenceNum)) diff --git a/src/pipeline/datatype/SpatialLocationCalculatorDataBindings.cpp b/src/pipeline/datatype/SpatialLocationCalculatorDataBindings.cpp index 98ff81754..9e0af7316 100644 --- a/src/pipeline/datatype/SpatialLocationCalculatorDataBindings.cpp +++ b/src/pipeline/datatype/SpatialLocationCalculatorDataBindings.cpp @@ -50,9 +50,9 @@ void bind_spatiallocationcalculatordata(pybind11::module& m, void* pCallstack){ .def(py::init<>()) .def("getSpatialLocations", &SpatialLocationCalculatorData::getSpatialLocations, DOC(dai, SpatialLocationCalculatorData, getSpatialLocations)) .def_property("spatialLocations", [](SpatialLocationCalculatorData& loc) { return &loc.spatialLocations; }, [](SpatialLocationCalculatorData& loc, std::vector val) { loc.spatialLocations = val; }, DOC(dai, SpatialLocationCalculatorData, spatialLocations)) - .def("getTimestamp", &SpatialLocationCalculatorData::getTimestamp, DOC(dai, SpatialLocationCalculatorData, getTimestamp)) - .def("getTimestampDevice", &SpatialLocationCalculatorData::getTimestampDevice, DOC(dai, SpatialLocationCalculatorData, getTimestampDevice)) - .def("getSequenceNum", &SpatialLocationCalculatorData::getSequenceNum, DOC(dai, SpatialLocationCalculatorData, getSequenceNum)) + .def("getTimestamp", &SpatialLocationCalculatorData::getTimestamp, DOC(dai, Buffer, getTimestamp)) + .def("getTimestampDevice", &SpatialLocationCalculatorData::getTimestampDevice, DOC(dai, Buffer, getTimestampDevice)) + .def("getSequenceNum", &SpatialLocationCalculatorData::getSequenceNum, DOC(dai, Buffer, getSequenceNum)) .def("setTimestamp", &SpatialLocationCalculatorData::setTimestamp, DOC(dai, SpatialLocationCalculatorData, setTimestamp)) .def("setTimestampDevice", &SpatialLocationCalculatorData::setTimestampDevice, DOC(dai, SpatialLocationCalculatorData, setTimestampDevice)) .def("setSequenceNum", &SpatialLocationCalculatorData::setSequenceNum, DOC(dai, SpatialLocationCalculatorData, setSequenceNum)) diff --git a/src/pipeline/datatype/TrackedFeaturesBindings.cpp b/src/pipeline/datatype/TrackedFeaturesBindings.cpp index 391e2fe79..fce447561 100644 --- a/src/pipeline/datatype/TrackedFeaturesBindings.cpp +++ b/src/pipeline/datatype/TrackedFeaturesBindings.cpp @@ -53,9 +53,9 @@ void bind_trackedfeatures(pybind11::module& m, void* pCallstack){ trackedFeatures .def(py::init<>()) .def_property("trackedFeatures", [](TrackedFeatures& feat) { return &feat.trackedFeatures; }, [](TrackedFeatures& feat, std::vector val) { feat.trackedFeatures = val; }, DOC(dai, TrackedFeatures, trackedFeatures)) - .def("getTimestamp", &TrackedFeatures::getTimestamp, DOC(dai, TrackedFeatures, getTimestamp)) - .def("getTimestampDevice", &TrackedFeatures::getTimestampDevice, DOC(dai, TrackedFeatures, getTimestampDevice)) - .def("getSequenceNum", &TrackedFeatures::getSequenceNum, DOC(dai, TrackedFeatures, getSequenceNum)) + .def("getTimestamp", &TrackedFeatures::getTimestamp, DOC(dai, Buffer, getTimestamp)) + .def("getTimestampDevice", &TrackedFeatures::getTimestampDevice, DOC(dai, Buffer, getTimestampDevice)) + .def("getSequenceNum", &TrackedFeatures::getSequenceNum, DOC(dai, Buffer, getSequenceNum)) .def("setTimestamp", &TrackedFeatures::setTimestamp, DOC(dai, TrackedFeatures, setTimestamp)) .def("setTimestampDevice", &TrackedFeatures::setTimestampDevice, DOC(dai, TrackedFeatures, setTimestampDevice)) .def("setSequenceNum", &TrackedFeatures::setSequenceNum, DOC(dai, TrackedFeatures, setSequenceNum)) diff --git a/src/pipeline/datatype/TrackletsBindings.cpp b/src/pipeline/datatype/TrackletsBindings.cpp index 353b61a04..b4c744eb9 100644 --- a/src/pipeline/datatype/TrackletsBindings.cpp +++ b/src/pipeline/datatype/TrackletsBindings.cpp @@ -62,9 +62,9 @@ void bind_tracklets(pybind11::module& m, void* pCallstack){ tracklets .def(py::init<>()) .def_property("tracklets", [](Tracklets& track) { return &track.tracklets; }, [](Tracklets& track, std::vector val) { track.tracklets = val; }, DOC(dai, Tracklets, tracklets)) - .def("getTimestamp", &Tracklets::getTimestamp, DOC(dai, Tracklets, getTimestamp)) - .def("getTimestampDevice", &Tracklets::getTimestampDevice, DOC(dai, Tracklets, getTimestampDevice)) - .def("getSequenceNum", &Tracklets::getSequenceNum, DOC(dai, Tracklets, getSequenceNum)) + .def("getTimestamp", &Tracklets::getTimestamp, DOC(dai, Buffer, getTimestamp)) + .def("getTimestampDevice", &Tracklets::getTimestampDevice, DOC(dai, Buffer, getTimestampDevice)) + .def("getSequenceNum", &Tracklets::getSequenceNum, DOC(dai, Buffer, getSequenceNum)) .def("setTimestamp", &Tracklets::setTimestamp, DOC(dai, Tracklets, setTimestamp)) .def("setTimestampDevice", &Tracklets::setTimestampDevice, DOC(dai, Tracklets, setTimestampDevice)) .def("setSequenceNum", &Tracklets::setSequenceNum, DOC(dai, Tracklets, setSequenceNum)) From d8ae04760fb67c5bd75ad5146087e6d0e7d8e7cd Mon Sep 17 00:00:00 2001 From: SzabolcsGergely Date: Fri, 1 Sep 2023 14:09:05 +0300 Subject: [PATCH 58/87] Fix windows build --- depthai-core | 2 +- src/pipeline/datatype/AprilTagsBindings.cpp | 6 +++--- src/pipeline/datatype/ImgDetectionsBindings.cpp | 6 +++--- src/pipeline/datatype/ImgFrameBindings.cpp | 10 +++++----- src/pipeline/datatype/NNDataBindings.cpp | 6 +++--- src/pipeline/datatype/SpatialImgDetectionsBindings.cpp | 6 +++--- .../datatype/SpatialLocationCalculatorDataBindings.cpp | 6 +++--- src/pipeline/datatype/TrackedFeaturesBindings.cpp | 6 +++--- src/pipeline/datatype/TrackletsBindings.cpp | 6 +++--- 9 files changed, 27 insertions(+), 27 deletions(-) diff --git a/depthai-core b/depthai-core index 47055d527..1c5fe7d07 160000 --- a/depthai-core +++ b/depthai-core @@ -1 +1 @@ -Subproject commit 47055d5270b13adbb740cb1a897b7f9fe7a98aa5 +Subproject commit 1c5fe7d075e67d5c722ab2851218582404eac35a diff --git a/src/pipeline/datatype/AprilTagsBindings.cpp b/src/pipeline/datatype/AprilTagsBindings.cpp index 66041e92a..778b96faa 100644 --- a/src/pipeline/datatype/AprilTagsBindings.cpp +++ b/src/pipeline/datatype/AprilTagsBindings.cpp @@ -48,9 +48,9 @@ void bind_apriltags(pybind11::module& m, void* pCallstack){ aprilTags .def(py::init<>()) .def_property("aprilTags", [](AprilTags& det) { return &det.aprilTags; }, [](AprilTags& det, std::vector val) { det.aprilTags = val; }) - .def("getTimestamp", &AprilTags::getTimestamp, DOC(dai, Buffer, getTimestamp)) - .def("getTimestampDevice", &AprilTags::getTimestampDevice, DOC(dai, Buffer, getTimestampDevice)) - .def("getSequenceNum", &AprilTags::getSequenceNum, DOC(dai, Buffer, getSequenceNum)) + .def("getTimestamp", &AprilTags::Buffer::getTimestamp, DOC(dai, Buffer, getTimestamp)) + .def("getTimestampDevice", &AprilTags::Buffer::getTimestampDevice, DOC(dai, Buffer, getTimestampDevice)) + .def("getSequenceNum", &AprilTags::Buffer::getSequenceNum, DOC(dai, Buffer, getSequenceNum)) .def("setTimestamp", &AprilTags::setTimestamp, DOC(dai, AprilTags, setTimestamp)) .def("setTimestampDevice", &AprilTags::setTimestampDevice, DOC(dai, AprilTags, setTimestampDevice)) .def("setSequenceNum", &AprilTags::setSequenceNum, DOC(dai, AprilTags, setSequenceNum)) diff --git a/src/pipeline/datatype/ImgDetectionsBindings.cpp b/src/pipeline/datatype/ImgDetectionsBindings.cpp index 6d8d01b4b..0ac6d2838 100644 --- a/src/pipeline/datatype/ImgDetectionsBindings.cpp +++ b/src/pipeline/datatype/ImgDetectionsBindings.cpp @@ -74,9 +74,9 @@ void bind_imgdetections(pybind11::module& m, void* pCallstack){ imgDetections .def(py::init<>(), DOC(dai, ImgDetections, ImgDetections)) .def_property("detections", [](ImgDetections& det) { return &det.detections; }, [](ImgDetections& det, std::vector val) { det.detections = val; }, DOC(dai, ImgDetections, detections)) - .def("getTimestamp", &ImgDetections::getTimestamp, DOC(dai, Buffer, getTimestamp)) - .def("getTimestampDevice", &ImgDetections::getTimestampDevice, DOC(dai, Buffer, getTimestampDevice)) - .def("getSequenceNum", &ImgDetections::getSequenceNum, DOC(dai, Buffer, getSequenceNum)) + .def("getTimestamp", &ImgDetections::Buffer::getTimestamp, DOC(dai, Buffer, getTimestamp)) + .def("getTimestampDevice", &ImgDetections::Buffer::getTimestampDevice, DOC(dai, Buffer, getTimestampDevice)) + .def("getSequenceNum", &ImgDetections::Buffer::getSequenceNum, DOC(dai, Buffer, getSequenceNum)) .def("setTimestamp", &ImgDetections::setTimestamp, DOC(dai, ImgDetections, setTimestamp)) .def("setTimestampDevice", &ImgDetections::setTimestampDevice, DOC(dai, ImgDetections, setTimestampDevice)) .def("setSequenceNum", &ImgDetections::setSequenceNum, DOC(dai, ImgDetections, setSequenceNum)) diff --git a/src/pipeline/datatype/ImgFrameBindings.cpp b/src/pipeline/datatype/ImgFrameBindings.cpp index 00b86aba9..98c87d2b5 100644 --- a/src/pipeline/datatype/ImgFrameBindings.cpp +++ b/src/pipeline/datatype/ImgFrameBindings.cpp @@ -117,13 +117,13 @@ void bind_imgframe(pybind11::module& m, void* pCallstack){ imgFrame .def(py::init<>()) // getters - .def("getTimestamp", py::overload_cast<>(&ImgFrame::getTimestamp, py::const_), DOC(dai, Buffer, getTimestamp)) - .def("getTimestampDevice", py::overload_cast<>(&ImgFrame::getTimestampDevice, py::const_), DOC(dai, Buffer, getTimestampDevice)) - .def("getTimestamp", py::overload_cast(&ImgFrame::getTimestamp, py::const_), py::arg("offset"), DOC(dai, Buffer, getTimestamp)) - .def("getTimestampDevice", py::overload_cast(&ImgFrame::getTimestampDevice, py::const_), py::arg("offset"), DOC(dai, Buffer, getTimestampDevice)) + .def("getTimestamp", py::overload_cast<>(&ImgFrame::Buffer::getTimestamp, py::const_), DOC(dai, Buffer, getTimestamp)) + .def("getTimestampDevice", py::overload_cast<>(&ImgFrame::Buffer::getTimestampDevice, py::const_), DOC(dai, Buffer, getTimestampDevice)) + .def("getTimestamp", py::overload_cast(&ImgFrame::getTimestamp, py::const_), py::arg("offset"), DOC(dai, ImgFrame, getTimestamp)) + .def("getTimestampDevice", py::overload_cast(&ImgFrame::getTimestampDevice, py::const_), py::arg("offset"), DOC(dai, ImgFrame, getTimestampDevice)) + .def("getSequenceNum", &ImgFrame::Buffer::getSequenceNum, DOC(dai, Buffer, getSequenceNum)) .def("getInstanceNum", &ImgFrame::getInstanceNum, DOC(dai, ImgFrame, getInstanceNum)) .def("getCategory", &ImgFrame::getCategory, DOC(dai, ImgFrame, getCategory)) - .def("getSequenceNum", &ImgFrame::getSequenceNum, DOC(dai, Buffer, getSequenceNum)) .def("getWidth", &ImgFrame::getWidth, DOC(dai, ImgFrame, getWidth)) .def("getHeight", &ImgFrame::getHeight, DOC(dai, ImgFrame, getHeight)) .def("getType", &ImgFrame::getType, DOC(dai, ImgFrame, getType)) diff --git a/src/pipeline/datatype/NNDataBindings.cpp b/src/pipeline/datatype/NNDataBindings.cpp index 9d0bef497..16dab6838 100644 --- a/src/pipeline/datatype/NNDataBindings.cpp +++ b/src/pipeline/datatype/NNDataBindings.cpp @@ -125,9 +125,9 @@ void bind_nndata(pybind11::module& m, void* pCallstack){ .def("getFirstLayerUInt8", &NNData::getFirstLayerUInt8, DOC(dai, NNData, getFirstLayerUInt8)) .def("getFirstLayerFp16", &NNData::getFirstLayerFp16, DOC(dai, NNData, getFirstLayerFp16)) .def("getFirstLayerInt32", &NNData::getFirstLayerInt32, DOC(dai, NNData, getFirstLayerInt32)) - .def("getTimestamp", &NNData::getTimestamp, DOC(dai, Buffer, getTimestamp)) - .def("getTimestampDevice", &NNData::getTimestampDevice, DOC(dai, Buffer, getTimestampDevice)) - .def("getSequenceNum", &NNData::getSequenceNum, DOC(dai, Buffer, getSequenceNum)) + .def("getTimestamp", &NNData::Buffer::getTimestamp, DOC(dai, Buffer, getTimestamp)) + .def("getTimestampDevice", &NNData::Buffer::getTimestampDevice, DOC(dai, Buffer, getTimestampDevice)) + .def("getSequenceNum", &NNData::Buffer::getSequenceNum, DOC(dai, Buffer, getSequenceNum)) .def("setTimestamp", &NNData::setTimestamp, DOC(dai, NNData, setTimestamp)) .def("setTimestampDevice", &NNData::setTimestampDevice, DOC(dai, NNData, setTimestampDevice)) .def("setSequenceNum", &NNData::setSequenceNum, DOC(dai, NNData, setSequenceNum)) diff --git a/src/pipeline/datatype/SpatialImgDetectionsBindings.cpp b/src/pipeline/datatype/SpatialImgDetectionsBindings.cpp index 86866bc3e..a0017df97 100644 --- a/src/pipeline/datatype/SpatialImgDetectionsBindings.cpp +++ b/src/pipeline/datatype/SpatialImgDetectionsBindings.cpp @@ -70,9 +70,9 @@ void bind_spatialimgdetections(pybind11::module& m, void* pCallstack){ spatialImgDetections .def(py::init<>()) .def_property("detections", [](SpatialImgDetections& det) { return &det.detections; }, [](SpatialImgDetections& det, std::vector val) { det.detections = val; }) - .def("getTimestamp", &SpatialImgDetections::getTimestamp, DOC(dai, Buffer, getTimestamp)) - .def("getTimestampDevice", &SpatialImgDetections::getTimestampDevice, DOC(dai, Buffer, getTimestampDevice)) - .def("getSequenceNum", &SpatialImgDetections::getSequenceNum, DOC(dai, Buffer, getSequenceNum)) + .def("getTimestamp", &SpatialImgDetections::Buffer::getTimestamp, DOC(dai, Buffer, getTimestamp)) + .def("getTimestampDevice", &SpatialImgDetections::Buffer::getTimestampDevice, DOC(dai, Buffer, getTimestampDevice)) + .def("getSequenceNum", &SpatialImgDetections::Buffer::getSequenceNum, DOC(dai, Buffer, getSequenceNum)) .def("setTimestamp", &SpatialImgDetections::setTimestamp, DOC(dai, SpatialImgDetections, setTimestamp)) .def("setTimestampDevice", &SpatialImgDetections::setTimestampDevice, DOC(dai, SpatialImgDetections, setTimestampDevice)) .def("setSequenceNum", &SpatialImgDetections::setSequenceNum, DOC(dai, SpatialImgDetections, setSequenceNum)) diff --git a/src/pipeline/datatype/SpatialLocationCalculatorDataBindings.cpp b/src/pipeline/datatype/SpatialLocationCalculatorDataBindings.cpp index 9e0af7316..c02551b79 100644 --- a/src/pipeline/datatype/SpatialLocationCalculatorDataBindings.cpp +++ b/src/pipeline/datatype/SpatialLocationCalculatorDataBindings.cpp @@ -50,9 +50,9 @@ void bind_spatiallocationcalculatordata(pybind11::module& m, void* pCallstack){ .def(py::init<>()) .def("getSpatialLocations", &SpatialLocationCalculatorData::getSpatialLocations, DOC(dai, SpatialLocationCalculatorData, getSpatialLocations)) .def_property("spatialLocations", [](SpatialLocationCalculatorData& loc) { return &loc.spatialLocations; }, [](SpatialLocationCalculatorData& loc, std::vector val) { loc.spatialLocations = val; }, DOC(dai, SpatialLocationCalculatorData, spatialLocations)) - .def("getTimestamp", &SpatialLocationCalculatorData::getTimestamp, DOC(dai, Buffer, getTimestamp)) - .def("getTimestampDevice", &SpatialLocationCalculatorData::getTimestampDevice, DOC(dai, Buffer, getTimestampDevice)) - .def("getSequenceNum", &SpatialLocationCalculatorData::getSequenceNum, DOC(dai, Buffer, getSequenceNum)) + .def("getTimestamp", &SpatialLocationCalculatorData::Buffer::getTimestamp, DOC(dai, Buffer, getTimestamp)) + .def("getTimestampDevice", &SpatialLocationCalculatorData::Buffer::getTimestampDevice, DOC(dai, Buffer, getTimestampDevice)) + .def("getSequenceNum", &SpatialLocationCalculatorData::Buffer::getSequenceNum, DOC(dai, Buffer, getSequenceNum)) .def("setTimestamp", &SpatialLocationCalculatorData::setTimestamp, DOC(dai, SpatialLocationCalculatorData, setTimestamp)) .def("setTimestampDevice", &SpatialLocationCalculatorData::setTimestampDevice, DOC(dai, SpatialLocationCalculatorData, setTimestampDevice)) .def("setSequenceNum", &SpatialLocationCalculatorData::setSequenceNum, DOC(dai, SpatialLocationCalculatorData, setSequenceNum)) diff --git a/src/pipeline/datatype/TrackedFeaturesBindings.cpp b/src/pipeline/datatype/TrackedFeaturesBindings.cpp index fce447561..10056af43 100644 --- a/src/pipeline/datatype/TrackedFeaturesBindings.cpp +++ b/src/pipeline/datatype/TrackedFeaturesBindings.cpp @@ -53,9 +53,9 @@ void bind_trackedfeatures(pybind11::module& m, void* pCallstack){ trackedFeatures .def(py::init<>()) .def_property("trackedFeatures", [](TrackedFeatures& feat) { return &feat.trackedFeatures; }, [](TrackedFeatures& feat, std::vector val) { feat.trackedFeatures = val; }, DOC(dai, TrackedFeatures, trackedFeatures)) - .def("getTimestamp", &TrackedFeatures::getTimestamp, DOC(dai, Buffer, getTimestamp)) - .def("getTimestampDevice", &TrackedFeatures::getTimestampDevice, DOC(dai, Buffer, getTimestampDevice)) - .def("getSequenceNum", &TrackedFeatures::getSequenceNum, DOC(dai, Buffer, getSequenceNum)) + .def("getTimestamp", &TrackedFeatures::Buffer::getTimestamp, DOC(dai, Buffer, getTimestamp)) + .def("getTimestampDevice", &TrackedFeatures::Buffer::getTimestampDevice, DOC(dai, Buffer, getTimestampDevice)) + .def("getSequenceNum", &TrackedFeatures::Buffer::getSequenceNum, DOC(dai, Buffer, getSequenceNum)) .def("setTimestamp", &TrackedFeatures::setTimestamp, DOC(dai, TrackedFeatures, setTimestamp)) .def("setTimestampDevice", &TrackedFeatures::setTimestampDevice, DOC(dai, TrackedFeatures, setTimestampDevice)) .def("setSequenceNum", &TrackedFeatures::setSequenceNum, DOC(dai, TrackedFeatures, setSequenceNum)) diff --git a/src/pipeline/datatype/TrackletsBindings.cpp b/src/pipeline/datatype/TrackletsBindings.cpp index b4c744eb9..d80fd676b 100644 --- a/src/pipeline/datatype/TrackletsBindings.cpp +++ b/src/pipeline/datatype/TrackletsBindings.cpp @@ -62,9 +62,9 @@ void bind_tracklets(pybind11::module& m, void* pCallstack){ tracklets .def(py::init<>()) .def_property("tracklets", [](Tracklets& track) { return &track.tracklets; }, [](Tracklets& track, std::vector val) { track.tracklets = val; }, DOC(dai, Tracklets, tracklets)) - .def("getTimestamp", &Tracklets::getTimestamp, DOC(dai, Buffer, getTimestamp)) - .def("getTimestampDevice", &Tracklets::getTimestampDevice, DOC(dai, Buffer, getTimestampDevice)) - .def("getSequenceNum", &Tracklets::getSequenceNum, DOC(dai, Buffer, getSequenceNum)) + .def("getTimestamp", &Tracklets::Buffer::getTimestamp, DOC(dai, Buffer, getTimestamp)) + .def("getTimestampDevice", &Tracklets::Buffer::getTimestampDevice, DOC(dai, Buffer, getTimestampDevice)) + .def("getSequenceNum", &Tracklets::Buffer::getSequenceNum, DOC(dai, Buffer, getSequenceNum)) .def("setTimestamp", &Tracklets::setTimestamp, DOC(dai, Tracklets, setTimestamp)) .def("setTimestampDevice", &Tracklets::setTimestampDevice, DOC(dai, Tracklets, setTimestampDevice)) .def("setSequenceNum", &Tracklets::setSequenceNum, DOC(dai, Tracklets, setSequenceNum)) From bf7f3b24eccb0ed35100be1077cedd71047f09db Mon Sep 17 00:00:00 2001 From: Aniel Alexa Date: Wed, 6 Sep 2023 17:23:45 +0300 Subject: [PATCH 59/87] diagnostic tool that zip important info --- examples/CrashReport/capture_diagnostic.py | 93 ++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 examples/CrashReport/capture_diagnostic.py diff --git a/examples/CrashReport/capture_diagnostic.py b/examples/CrashReport/capture_diagnostic.py new file mode 100644 index 000000000..c9156e1a6 --- /dev/null +++ b/examples/CrashReport/capture_diagnostic.py @@ -0,0 +1,93 @@ +import depthai as dai +import zipfile +from json import dump , JSONEncoder +from os.path import exists +import os + +class CustomEncoder(JSONEncoder): + def default(self, obj): + # If the object has a `__str__` method, use it + if hasattr(obj, '__str__'): + return str(obj) + # Otherwise, use the default encoder's method + return super().default(obj) + +data = {} + +with dai.Device() as device: + + data['BootloaderVersion'] = device.getBootloaderVersion() + data['ConnectedIMU'] = device.getConnectedIMU() + data['imuFirmwareVersion'] = device.getIMUFirmwareVersion() + filename = 'metadata.json' + with open(filename, 'w') as f: + dump(data, f, indent=2, cls=CustomEncoder) + + + usercalibPath = 'userCalib.json' + try: + + with open(usercalibPath,'w') as f: + dump(device.readCalibration2().eepromToJson(), f, indent=2) + dump(device.readCalibrationRaw(), f, indent=4) + except Exception as ex: + + with open(usercalibPath,'w') as f: + dump(str(ex), f, indent=2) + + factcalibPath = 'factoryCalib.json' + try: + + with open(factcalibPath,'w') as f: + dump(device.readFactoryCalibration().eepromToJson(), f, indent=2) + dump(device.readFactoryCalibrationRaw(), f, indent=4) + except Exception as ex: + + with open(factcalibPath,'w') as f: + dump(str(ex), f, indent=2) + + zip_name = 'device_info.zip' + if device.hasCrashDump(): + crashDump = device.getCrashDump() + commitHash = crashDump.depthaiCommitHash + deviceId = crashDump.deviceId + + json = crashDump.serializeToJson() + + i = -1 + while True: + i += 1 + destPath = "crashDump_" + str(i) + "_" + deviceId + "_" + commitHash + ".json" + if exists(destPath): + continue + + with open(destPath, 'w', encoding='utf-8') as f: + dump(json, f, ensure_ascii=False, indent=4) + + print("Crash dump found on your device!") + print(f"Saved to {destPath}") + print("Please report to developers!") + with zipfile.ZipFile(zip_name, 'w') as z: + z.write(filename) + z.write(usercalibPath) + z.write(factcalibPath) + z.write(destPath) + os.remove(filename) + os.remove(usercalibPath) + os.remove(factcalibPath) + os.remove(destPath) + break + else: + with zipfile.ZipFile(zip_name, 'w') as z: + z.write(filename) + z.write(usercalibPath) + z.write(factcalibPath) + os.remove(filename) + os.remove(usercalibPath) + os.remove(factcalibPath) + print("There was no crash dump found on your device!") +# Zip the JSON file + + + +print("Please report to developers via forum with the zip attached!") From 06411b289d9dbd7c3adb058ec66bcbe9a1d6cb3b Mon Sep 17 00:00:00 2001 From: Aniel Alexa Date: Wed, 6 Sep 2023 17:52:17 +0300 Subject: [PATCH 60/87] no more saving jason file on disk --- examples/CrashReport/capture_diagnostic.py | 100 +++++++++------------ 1 file changed, 41 insertions(+), 59 deletions(-) diff --git a/examples/CrashReport/capture_diagnostic.py b/examples/CrashReport/capture_diagnostic.py index c9156e1a6..bcecf992d 100644 --- a/examples/CrashReport/capture_diagnostic.py +++ b/examples/CrashReport/capture_diagnostic.py @@ -1,93 +1,75 @@ import depthai as dai import zipfile -from json import dump , JSONEncoder -from os.path import exists -import os +from json import dump, JSONEncoder +import io class CustomEncoder(JSONEncoder): def default(self, obj): - # If the object has a `__str__` method, use it if hasattr(obj, '__str__'): return str(obj) - # Otherwise, use the default encoder's method return super().default(obj) data = {} with dai.Device() as device: - data['BootloaderVersion'] = device.getBootloaderVersion() data['ConnectedIMU'] = device.getConnectedIMU() data['imuFirmwareVersion'] = device.getIMUFirmwareVersion() - filename = 'metadata.json' - with open(filename, 'w') as f: - dump(data, f, indent=2, cls=CustomEncoder) + filenames = [ + "metadata.json", + "userCalib.json", + "factoryCalib.json" + + ] - usercalibPath = 'userCalib.json' + # Create in-memory file-like objects + metadata_io = io.StringIO() + usercalib_io = io.StringIO() + factcalib_io = io.StringIO() + crashDump_io = io.StringIO() + # Dump data into these in-memory files + dump(data, metadata_io, indent=2, cls=CustomEncoder) + try: - - with open(usercalibPath,'w') as f: - dump(device.readCalibration2().eepromToJson(), f, indent=2) - dump(device.readCalibrationRaw(), f, indent=4) + dump(device.readCalibration2().eepromToJson(), usercalib_io, indent=2) + dump(device.readCalibrationRaw(), usercalib_io, indent=4) except Exception as ex: - - with open(usercalibPath,'w') as f: - dump(str(ex), f, indent=2) + dump(str(ex), usercalib_io, indent=2) - factcalibPath = 'factoryCalib.json' try: - - with open(factcalibPath,'w') as f: - dump(device.readFactoryCalibration().eepromToJson(), f, indent=2) - dump(device.readFactoryCalibrationRaw(), f, indent=4) + dump(device.readFactoryCalibration().eepromToJson(), factcalib_io, indent=2) + dump(device.readFactoryCalibrationRaw(), factcalib_io, indent=4) except Exception as ex: - - with open(factcalibPath,'w') as f: - dump(str(ex), f, indent=2) + dump(str(ex), factcalib_io, indent=2) + + in_memory_files = [metadata_io, usercalib_io, factcalib_io] zip_name = 'device_info.zip' + + with zipfile.ZipFile(zip_name, 'w') as z: + # Zip in-memory file-like objects without writing to disk + for file_io, filename in zip(in_memory_files, filenames): + file_io.seek(0) # Reset cursor to the beginning of the file + z.writestr(filename, file_io.getvalue()) + if device.hasCrashDump(): crashDump = device.getCrashDump() commitHash = crashDump.depthaiCommitHash deviceId = crashDump.deviceId + filenames.append("crashreport_" + commitHash + "_" + deviceId + ".json") + dump(crashDump.serializeToJson(), crashDump_io, indent=2) + in_memory_files.append(crashDump_io) - json = crashDump.serializeToJson() - - i = -1 - while True: - i += 1 - destPath = "crashDump_" + str(i) + "_" + deviceId + "_" + commitHash + ".json" - if exists(destPath): - continue - - with open(destPath, 'w', encoding='utf-8') as f: - dump(json, f, ensure_ascii=False, indent=4) + with zipfile.ZipFile(zip_name, 'w') as z: + # Zip in-memory file-like objects without writing to disk + for file_io, filename in zip(in_memory_files, filenames): + file_io.seek(0) # Reset cursor to the beginning of the file + z.writestr(filename, file_io.getvalue()) - print("Crash dump found on your device!") - print(f"Saved to {destPath}") - print("Please report to developers!") - with zipfile.ZipFile(zip_name, 'w') as z: - z.write(filename) - z.write(usercalibPath) - z.write(factcalibPath) - z.write(destPath) - os.remove(filename) - os.remove(usercalibPath) - os.remove(factcalibPath) - os.remove(destPath) - break + print("Crash dump found on your device!") + print("Please report to developers!") else: - with zipfile.ZipFile(zip_name, 'w') as z: - z.write(filename) - z.write(usercalibPath) - z.write(factcalibPath) - os.remove(filename) - os.remove(usercalibPath) - os.remove(factcalibPath) print("There was no crash dump found on your device!") -# Zip the JSON file - - print("Please report to developers via forum with the zip attached!") From ae6aabd244ec2ec3ddb9204736810bbae52c9dc9 Mon Sep 17 00:00:00 2001 From: Aniel Alexa Date: Wed, 6 Sep 2023 17:56:07 +0300 Subject: [PATCH 61/87] minor fixes --- examples/CrashReport/capture_diagnostic.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/examples/CrashReport/capture_diagnostic.py b/examples/CrashReport/capture_diagnostic.py index bcecf992d..5962406f0 100644 --- a/examples/CrashReport/capture_diagnostic.py +++ b/examples/CrashReport/capture_diagnostic.py @@ -19,8 +19,7 @@ def default(self, obj): filenames = [ "metadata.json", "userCalib.json", - "factoryCalib.json" - + "factoryCalib.json" ] # Create in-memory file-like objects @@ -68,8 +67,7 @@ def default(self, obj): z.writestr(filename, file_io.getvalue()) print("Crash dump found on your device!") - print("Please report to developers!") else: print("There was no crash dump found on your device!") -print("Please report to developers via forum with the zip attached!") +print("Please report to developers with the zip attached!") From b1d6d304dd83ce5f4795a4955eeb4ccd351420c7 Mon Sep 17 00:00:00 2001 From: Martin Peterlin Date: Tue, 12 Sep 2023 23:14:28 +0200 Subject: [PATCH 62/87] Added getProductName and tweaked getDeviceName --- depthai-core | 2 +- src/CalibrationHandlerBindings.cpp | 3 +++ src/DeviceBindings.cpp | 1 + src/pipeline/CommonBindings.cpp | 7 ++++--- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/depthai-core b/depthai-core index 1c5fe7d07..d841b96fc 160000 --- a/depthai-core +++ b/depthai-core @@ -1 +1 @@ -Subproject commit 1c5fe7d075e67d5c722ab2851218582404eac35a +Subproject commit d841b96fc3290092ef664c9634dd1b3efaf3764c diff --git a/src/CalibrationHandlerBindings.cpp b/src/CalibrationHandlerBindings.cpp index 52f7bf3ae..69abd737a 100644 --- a/src/CalibrationHandlerBindings.cpp +++ b/src/CalibrationHandlerBindings.cpp @@ -62,6 +62,9 @@ void CalibrationHandlerBindings::bind(pybind11::module& m, void* pCallstack){ .def("setBoardInfo", py::overload_cast(&CalibrationHandler::setBoardInfo), py::arg("boardName"), py::arg("boardRev"), DOC(dai, CalibrationHandler, setBoardInfo)) .def("setBoardInfo", py::overload_cast(&CalibrationHandler::setBoardInfo), py::arg("productName"), py::arg("boardName"), py::arg("boardRev"), py::arg("boardConf"), py::arg("hardwareConf"), py::arg("batchName"), py::arg("batchTime"), py::arg("boardOptions"), py::arg("boardCustom") = "", DOC(dai, CalibrationHandler, setBoardInfo, 2)) + .def("setBoardInfo", py::overload_cast(&CalibrationHandler::setBoardInfo), py::arg("deviceName"), py::arg("productName"), py::arg("boardName"), py::arg("boardRev"), py::arg("boardConf"), py::arg("hardwareConf"), py::arg("batchName"), py::arg("batchTime"), py::arg("boardOptions"), py::arg("boardCustom") = "", DOC(dai, CalibrationHandler, setBoardInfo, 3)) + + .def("setDeviceName", &CalibrationHandler::setDeviceName, py::arg("deviceName"), DOC(dai, CalibrationHandler, setDeviceName)) .def("setProductName", &CalibrationHandler::setProductName, py::arg("productName"), DOC(dai, CalibrationHandler, setProductName)) .def("setCameraIntrinsics", py::overload_cast>, Size2f>(&CalibrationHandler::setCameraIntrinsics), py::arg("cameraId"), py::arg("intrinsics"), py::arg("frameSize"), DOC(dai, CalibrationHandler, setCameraIntrinsics)) diff --git a/src/DeviceBindings.cpp b/src/DeviceBindings.cpp index afd3c99eb..447ab9f81 100644 --- a/src/DeviceBindings.cpp +++ b/src/DeviceBindings.cpp @@ -653,6 +653,7 @@ void DeviceBindings::bind(pybind11::module& m, void* pCallstack){ .def("setTimesync", [](DeviceBase& d, std::chrono::milliseconds p, int s, bool r) { py::gil_scoped_release release; return d.setTimesync(p,s,r); }, DOC(dai, DeviceBase, setTimesync)) .def("setTimesync", [](DeviceBase& d, bool e) { py::gil_scoped_release release; return d.setTimesync(e); }, py::arg("enable"), DOC(dai, DeviceBase, setTimesync, 2)) .def("getDeviceName", [](DeviceBase& d) { std::string name; { py::gil_scoped_release release; name = d.getDeviceName(); } return py::bytes(name).attr("decode")("utf-8", "replace"); }, DOC(dai, DeviceBase, getDeviceName)) + .def("getProductName", [](DeviceBase& d) { std::string name; { py::gil_scoped_release release; name = d.getProductName(); } return py::bytes(name).attr("decode")("utf-8", "replace"); }, DOC(dai, DeviceBase, getProductName)) ; diff --git a/src/pipeline/CommonBindings.cpp b/src/pipeline/CommonBindings.cpp index eaeb0bab0..a244b753f 100644 --- a/src/pipeline/CommonBindings.cpp +++ b/src/pipeline/CommonBindings.cpp @@ -64,7 +64,7 @@ void CommonBindings::bind(pybind11::module& m, void* pCallstack){ py::enum_ frameEvent(m, "FrameEvent", DOC(dai, FrameEvent)); py::class_ profilingData(m, "ProfilingData", DOC(dai, ProfilingData)); py::enum_ interpolation(m, "Interpolation", DOC(dai, Interpolation)); - + /////////////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////////// @@ -295,7 +295,8 @@ void CommonBindings::bind(pybind11::module& m, void* pCallstack){ .def_readwrite("boardConf", &EepromData::boardConf) .def_readwrite("hardwareConf", &EepromData::hardwareConf) .def_readwrite("productName", &EepromData::productName) - .def_readwrite("batchName", &EepromData::batchName) + // .def_readwrite("batchName", &EepromData::batchName) + .def_readwrite("deviceName", &EepromData::deviceName) .def_readwrite("batchTime", &EepromData::batchTime) .def_readwrite("boardOptions", &EepromData::boardOptions) .def_readwrite("cameraData", &EepromData::cameraData) @@ -379,7 +380,7 @@ void CommonBindings::bind(pybind11::module& m, void* pCallstack){ .value("READOUT_START", FrameEvent::READOUT_START) .value("READOUT_END", FrameEvent::READOUT_END) ; - + interpolation .value("BILINEAR", Interpolation::BILINEAR) .value("BICUBIC", Interpolation::BICUBIC) From bc2f4ccd5bd099174d678a619badd3d119ebceaa Mon Sep 17 00:00:00 2001 From: Martin Peterlin Date: Wed, 13 Sep 2023 15:31:55 +0200 Subject: [PATCH 63/87] Added some bindings for updated EEPROM and FW --- depthai-core | 2 +- src/pipeline/CommonBindings.cpp | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/depthai-core b/depthai-core index d841b96fc..8699c189f 160000 --- a/depthai-core +++ b/depthai-core @@ -1 +1 @@ -Subproject commit d841b96fc3290092ef664c9634dd1b3efaf3764c +Subproject commit 8699c189fe70250ae2f1b072dcfad60e0ca8e918 diff --git a/src/pipeline/CommonBindings.cpp b/src/pipeline/CommonBindings.cpp index a244b753f..6025f8bc6 100644 --- a/src/pipeline/CommonBindings.cpp +++ b/src/pipeline/CommonBindings.cpp @@ -262,6 +262,7 @@ void CommonBindings::bind(pybind11::module& m, void* pCallstack){ .def_readwrite("rectifiedRotationRight", &StereoRectification::rectifiedRotationRight) .def_readwrite("leftCameraSocket", &StereoRectification::leftCameraSocket) .def_readwrite("rightCameraSocket", &StereoRectification::rightCameraSocket) + .def_readwrite("verticalCameraSocket", &StereoRectification::verticalCameraSocket) ; // Extrinsics @@ -303,6 +304,9 @@ void CommonBindings::bind(pybind11::module& m, void* pCallstack){ .def_readwrite("stereoRectificationData", &EepromData::stereoRectificationData) .def_readwrite("imuExtrinsics", &EepromData::imuExtrinsics) .def_readwrite("miscellaneousData", &EepromData::miscellaneousData) + .def_readwrite("housingExtrinsics", &EepromData::housingExtrinsics) + .def_readwrite("stereoUseSpecTranslation", &EepromData::stereoUseSpecTranslation) + .def_readwrite("stereoEnableDistortionCorrection", &EepromData::stereoEnableDistortionCorrection) ; // UsbSpeed usbSpeed From c0e05a95014f3a8c0c21c213b34a9ee1f4f93991 Mon Sep 17 00:00:00 2001 From: asahtik Date: Fri, 15 Sep 2023 09:13:25 +0200 Subject: [PATCH 64/87] Bump core --- depthai-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/depthai-core b/depthai-core index 67aaedada..3a902393c 160000 --- a/depthai-core +++ b/depthai-core @@ -1 +1 @@ -Subproject commit 67aaedada6adddc0a6f9c37f780c3aa3e9961ef8 +Subproject commit 3a902393c99f77b8a5d961bb5375e5543eaf2c7a From 937a88aabbd85c9335ff22fd430b0c7773b67538 Mon Sep 17 00:00:00 2001 From: alex-luxonis Date: Fri, 15 Sep 2023 14:57:44 +0300 Subject: [PATCH 65/87] FW, ColorCamera: add new almost-full-FOV resolutions for IMX378/477, scaled: 1352x1012, 4lane-only, 1/3 scaling 2024x1520, 1/2 binning --- depthai-core | 2 +- src/pipeline/node/ColorCameraBindings.cpp | 2 ++ utilities/cam_test.py | 4 +++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/depthai-core b/depthai-core index 1c5fe7d07..57de7c0ca 160000 --- a/depthai-core +++ b/depthai-core @@ -1 +1 @@ -Subproject commit 1c5fe7d075e67d5c722ab2851218582404eac35a +Subproject commit 57de7c0cafd5724e10adce3ca8f69c8230ac6e9c diff --git a/src/pipeline/node/ColorCameraBindings.cpp b/src/pipeline/node/ColorCameraBindings.cpp index fb345d221..beee4587b 100644 --- a/src/pipeline/node/ColorCameraBindings.cpp +++ b/src/pipeline/node/ColorCameraBindings.cpp @@ -43,6 +43,8 @@ void bind_colorcamera(pybind11::module& m, void* pCallstack){ .value("THE_720_P", ColorCameraProperties::SensorResolution::THE_720_P) .value("THE_800_P", ColorCameraProperties::SensorResolution::THE_800_P) .value("THE_1440X1080", ColorCameraProperties::SensorResolution::THE_1440X1080) + .value("THE_1352X1012", ColorCameraProperties::SensorResolution::THE_1352X1012) + .value("THE_2024X1520", ColorCameraProperties::SensorResolution::THE_2024X1520) ; colorCameraPropertiesColorOrder diff --git a/utilities/cam_test.py b/utilities/cam_test.py index b3e478830..6ee7a9546 100755 --- a/utilities/cam_test.py +++ b/utilities/cam_test.py @@ -67,7 +67,7 @@ def socket_type_pair(arg): "E.g: -cams rgb,m right,c . Default: rgb,c left,m right,m camd,c") parser.add_argument('-mres', '--mono-resolution', type=int, default=800, choices={480, 400, 720, 800}, help="Select mono camera resolution (height). Default: %(default)s") -parser.add_argument('-cres', '--color-resolution', default='1080', choices={'720', '800', '1080', '1200', '4k', '5mp', '12mp', '13mp', '48mp'}, +parser.add_argument('-cres', '--color-resolution', default='1080', choices={'720', '800', '1080', '1012', '1200', '1520', '4k', '5mp', '12mp', '13mp', '48mp'}, help="Select color camera resolution / height. Default: %(default)s") parser.add_argument('-rot', '--rotate', const='all', choices={'all', 'rgb', 'mono'}, nargs="?", help="Which cameras to rotate 180 degrees. All if not filtered") @@ -159,7 +159,9 @@ def socket_type_pair(arg): '720': dai.ColorCameraProperties.SensorResolution.THE_720_P, '800': dai.ColorCameraProperties.SensorResolution.THE_800_P, '1080': dai.ColorCameraProperties.SensorResolution.THE_1080_P, + '1012': dai.ColorCameraProperties.SensorResolution.THE_1352X1012, '1200': dai.ColorCameraProperties.SensorResolution.THE_1200_P, + '1520': dai.ColorCameraProperties.SensorResolution.THE_2024X1520, '4k': dai.ColorCameraProperties.SensorResolution.THE_4_K, '5mp': dai.ColorCameraProperties.SensorResolution.THE_5_MP, '12mp': dai.ColorCameraProperties.SensorResolution.THE_12_MP, From 1ffd3fd73bb78519842078902a2042ea2e6758e4 Mon Sep 17 00:00:00 2001 From: TheMarpe Date: Thu, 21 Sep 2023 13:02:10 +0200 Subject: [PATCH 66/87] Updated pybind11_mkdoc, to fix CI issues --- docs/requirements_mkdoc.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/requirements_mkdoc.txt b/docs/requirements_mkdoc.txt index 700047e08..e2ff4511a 100644 --- a/docs/requirements_mkdoc.txt +++ b/docs/requirements_mkdoc.txt @@ -1 +1,2 @@ -git+https://github.com/luxonis/pybind11_mkdoc.git@59746f8d1645c9f00ebfb534186334d0154b5bd6 +git+https://github.com/luxonis/pybind11_mkdoc.git@da6c64251a0ebbc3ffc007477a0b9c9f20cac165 +libclang==16.0.6 From aceaffe37f2b1c474a3f91f006d9a7bfd60ea814 Mon Sep 17 00:00:00 2001 From: TheMarpe Date: Thu, 21 Sep 2023 13:30:17 +0200 Subject: [PATCH 67/87] Updated detection of pybind11_mkdoc being present --- .github/workflows/main.yml | 1 - cmake/pybind11-mkdoc.cmake | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index eed6deefa..79889f628 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -49,7 +49,6 @@ jobs: run: | python -m pip install --upgrade pip sudo apt install libusb-1.0-0-dev - python -m pip install clang==14.0 --force-reinstall python -m pip install -r docs/requirements_mkdoc.txt - name: Configure project run: cmake -S . -B build -DDEPTHAI_PYTHON_FORCE_DOCSTRINGS=ON -DDEPTHAI_PYTHON_DOCSTRINGS_OUTPUT="$PWD/docstrings/depthai_python_docstring.hpp" diff --git a/cmake/pybind11-mkdoc.cmake b/cmake/pybind11-mkdoc.cmake index 271926d08..33432b361 100644 --- a/cmake/pybind11-mkdoc.cmake +++ b/cmake/pybind11-mkdoc.cmake @@ -28,7 +28,7 @@ function(pybind11_mkdoc_setup_internal target output_path mkdoc_headers enforce) # Execute module pybind11_mkdoc to check if present message(STATUS "Checking for pybind11_mkdoc") - execute_process(COMMAND ${PYTHON_EXECUTABLE} -m ${PYBIND11_MKDOC_MODULE_NAME} RESULT_VARIABLE error OUTPUT_QUIET ERROR_QUIET) + execute_process(COMMAND ${PYTHON_EXECUTABLE} -m ${PYBIND11_MKDOC_MODULE_NAME} --help RESULT_VARIABLE error OUTPUT_QUIET ERROR_QUIET) if(error) set(message "Checking for pybind11_mkdoc - not found, docstrings not available") if(NOT enforce) From e7c7027ca1b74a34deba02e03117acb4179208e0 Mon Sep 17 00:00:00 2001 From: Erol444 Date: Fri, 22 Sep 2023 16:29:06 +0200 Subject: [PATCH 68/87] Update standalone mode docs --- docs/source/tutorials/standalone_mode.rst | 29 +++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/docs/source/tutorials/standalone_mode.rst b/docs/source/tutorials/standalone_mode.rst index 95f4440a5..8bc6464ab 100644 --- a/docs/source/tutorials/standalone_mode.rst +++ b/docs/source/tutorials/standalone_mode.rst @@ -1,13 +1,33 @@ Standalone mode =============== -**Standalone / Hostless / On-The-Edge mode** means that the OAK camera isn't connected to a host computer. This can -be achieved by first :ref:`flashing the bootloader ` and then :ref:`flashing the pipeline ` -and assets (NN models) to the OAK's flash memory. +**Standalone / Hostless / On-The-Edge mode** means that the camera starts the (:ref:`flashed `) application as soon as it gets power, without being connected to any particular host computer. +This is useful for applications where camera is stationary and just inspecting the world and providing analytics (eg. people/vehicle counting, LPR, fall detection, etc.) + +Usually, this mode is also more robust to any instabilities (eg. networking issues, where connection between camera and host computer would drop), as application will restart automatically. Standalone mode is **only possible on OAKs that have on-board flash** memory, which are currently `OAK POE `__ and OAK IOT camera models. +**Scenarios** when standalone mode is particularly useful: + +- People/vehicle/object tracking and counting (People counting `demo here `__). Each camera can do its own counting (inside Script node), and only send final count (eg. every hour) to some server via MQTT/HTTP. +- License Plate Recognition (LPR) camera (`demo here `__). Each camera does vehicle detection, license plate detection and LPR, and only reports license plate (in string) to a server. +- Fall detection for elderly people (`demo here `__). Each camera tracks people and checks their poses for any anomalies (eg. person falling down). If anomaly is detected, it sends an alert to a server. + +In case you already have a computer on-board (eg. a robot/drone), standalone mode isn't as useful, and just adds extra complexity. + +Communication with the camera +############################# + +To "communicate" with the outside world (eg. a server), POE cameras can use :ref:`Script` node to send/receive networking packets (HTTP/TCP/UDP...). Here are a few examples: + +- `TCP streaming `__ (camera being either server or client) +- `HTTP server `__ +- `HTTP client `__ +- `MQTT client `__ + + Converting a demo to standalone mode #################################### @@ -37,8 +57,9 @@ or script node, as mentioned above. Flash the bootloader #################### +To run the application on the camera, a `Bootloader` is required. Note that bootloader is already flashed on all POE cameras, as it's also required for network booting. The :ref:`Bootloader` is packaged together with the depthai, so if you have the latest depthai version, you can flash the -latest bootloader version. To flash the bootloader, we suggest using the :ref:`Device Manager`. To view the API code behind +latest bootloader version. To flash the latest bootloader, we suggest using the :ref:`Device Manager`. To view the API code behind it, see :ref:`Flash Bootloader` example code. Flash the pipeline From f4d598e09bf864441ecbbfb7de9c403505291a4a Mon Sep 17 00:00:00 2001 From: Martin Peterlin Date: Fri, 22 Sep 2023 22:50:42 +0200 Subject: [PATCH 69/87] Timesync improvements --- depthai-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/depthai-core b/depthai-core index b6f35d3b8..eba456e3d 160000 --- a/depthai-core +++ b/depthai-core @@ -1 +1 @@ -Subproject commit b6f35d3b8636cd328594147e6abd33e0a9b686ad +Subproject commit eba456e3d0b9cf95d881a7895570e8f61f78149c From a66f1b3f0ce2dbbb1335f81469426c0312c70e12 Mon Sep 17 00:00:00 2001 From: Martin Peterlin Date: Fri, 22 Sep 2023 22:51:26 +0200 Subject: [PATCH 70/87] [FW] OAK-D SR PoE and ToF improvements --- depthai-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/depthai-core b/depthai-core index eba456e3d..052603c08 160000 --- a/depthai-core +++ b/depthai-core @@ -1 +1 @@ -Subproject commit eba456e3d0b9cf95d881a7895570e8f61f78149c +Subproject commit 052603c088b4268c108f6a2bf7a8012accc599eb From 21d47177540d445cc3bd70d66e5b7813fe453a9a Mon Sep 17 00:00:00 2001 From: Martin Peterlin Date: Fri, 22 Sep 2023 22:51:41 +0200 Subject: [PATCH 71/87] Fixed incorrect warp in issue --- depthai-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/depthai-core b/depthai-core index 052603c08..b1e9c90ca 160000 --- a/depthai-core +++ b/depthai-core @@ -1 +1 @@ -Subproject commit 052603c088b4268c108f6a2bf7a8012accc599eb +Subproject commit b1e9c90cace41d661e6060b5a2c7eabd993eef19 From 63461c6d93df664cc2ee5a83d6c49a77cc266f04 Mon Sep 17 00:00:00 2001 From: Martin Peterlin Date: Sat, 23 Sep 2023 00:23:57 +0200 Subject: [PATCH 72/87] [FW] Improved OV9782 fps handling --- depthai-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/depthai-core b/depthai-core index b1e9c90ca..b92854229 160000 --- a/depthai-core +++ b/depthai-core @@ -1 +1 @@ -Subproject commit b1e9c90cace41d661e6060b5a2c7eabd993eef19 +Subproject commit b928542294770e42c1561202cefa0e8a0afc338c From 994e1926a41255e8b434bf2f717b197c43fae729 Mon Sep 17 00:00:00 2001 From: Erol444 Date: Sat, 23 Sep 2023 12:37:33 +0200 Subject: [PATCH 73/87] Update low latency docs, added usb comparisons --- docs/source/tutorials/low-latency.rst | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/docs/source/tutorials/low-latency.rst b/docs/source/tutorials/low-latency.rst index 4fa18f94f..631b77bed 100644 --- a/docs/source/tutorials/low-latency.rst +++ b/docs/source/tutorials/low-latency.rst @@ -4,14 +4,14 @@ Low Latency These tables show what performance you can expect from **USB 3.2** Gen 1 (5 Gbps) connection with an OAK camera. XLink chunking was disabled for these tests (:code:`pipeline.setXLinkChunkSize(0)`). For an example code, see :ref:`Latency measurement`. -.. list-table:: +.. list-table:: USB 5gbps latency :header-rows: 1 * - What - Resolution - FPS - FPS set - - Time-to-Host [ms] + - Latency [ms] - Bandwidth - Histogram * - Color (isp) @@ -43,9 +43,9 @@ disabled for these tests (:code:`pipeline.setXLinkChunkSize(0)`). For an example - 246 Mbps - `link `__ -Below are the same tests, but with **OAK PoE** camera, which uses Gigabit ethernet link. The camera was connected directly to the computer, +Below are the same tests, but also with **OAK PoE** camera, which uses Gigabit ethernet link. The camera was connected directly to the computer, without any switches or routers in between. Power was supplied via M8 connector. `oak_bandwidth_test.py `__ results: 797 mbps downlink, 264 mbps uplink. -`oak_latency_test.py `__ results: Average: 5.2 ms, Std: 6.2 +`oak_latency_test.py `__ results: Average: 5.2 ms, Std: 6.2. .. list-table:: :header-rows: 1 @@ -54,41 +54,49 @@ without any switches or routers in between. Power was supplied via M8 connector. - Resolution - FPS - FPS set - - Time-to-Host [ms] + - PoE Latency [ms] + - USB Latency [ms] - Bandwidth * - Color (isp) - 1080P - 25 - 25 - 51 + - 19.2 Std: 0.25 - 622 Mbps * - Color (isp) - 4K - 8 - 8 - 148 + - 19.4 Std: 0.6 - 530 Mbps * - Color (isp) - 4K - 8.5 - 10 - 530 + - 19.4 Std: 0.6 - 663 Mbps * - Mono - 400P - 90 - 90 - - Avrg: 12 (Std: 5.0) + - 12 Std: 5.0 + - 8 Std: 0.47 - 184 Mbps * - Mono - 400P - 110 - 110 - - Avrg: 16 (Std: 9.4) + - 16 Std: 9.4 + - 8 Std: 0.45 - 225 Mbps +We set lower FPS for the POE measurements due to bandwidth constraints. For example, 4K 8 FPS was 150ms latency, while +4K 10FPS had 530ms latency, as link was saturated. -- **Time-to-Host** is measured time between frame timestamp (:code:`imgFrame.getTimestamp()`) and host timestamp when the frame is received (:code:`dai.Clock.now()`). +- **Latency** is measured time between frame timestamp (:code:`imgFrame.getTimestamp()`) and host timestamp when the frame is received (:code:`dai.Clock.now()`). - **Histogram** shows how much Time-to-Host varies frame to frame. Y axis represents number of frame that occurred at that time while the X axis represents microseconds. - **Bandwidth** is calculated bandwidth required to stream specified frames at specified FPS. From fdc9c4272c0dec13d4e991bf7040e8a2ded43e1f Mon Sep 17 00:00:00 2001 From: Erol444 Date: Sat, 23 Sep 2023 12:55:32 +0200 Subject: [PATCH 74/87] Update values, as I was testing with ov9782 --- docs/source/tutorials/low-latency.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/tutorials/low-latency.rst b/docs/source/tutorials/low-latency.rst index 631b77bed..f08a13f58 100644 --- a/docs/source/tutorials/low-latency.rst +++ b/docs/source/tutorials/low-latency.rst @@ -62,21 +62,21 @@ without any switches or routers in between. Power was supplied via M8 connector. - 25 - 25 - 51 - - 19.2 Std: 0.25 + - 33 Std: 0.8 - 622 Mbps * - Color (isp) - 4K - 8 - 8 - 148 - - 19.4 Std: 0.6 + - 80 Std: 1.2 - 530 Mbps * - Color (isp) - 4K - 8.5 - 10 - 530 - - 19.4 Std: 0.6 + - 80 Std: 1.3 - 663 Mbps * - Mono - 400P From e1d98db0be3b28fdcbbb2b1fa5b4460f91004b54 Mon Sep 17 00:00:00 2001 From: Erol444 Date: Sat, 23 Sep 2023 13:02:13 +0200 Subject: [PATCH 75/87] Added usb 4k at 26fps latency --- docs/source/tutorials/low-latency.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/source/tutorials/low-latency.rst b/docs/source/tutorials/low-latency.rst index f08a13f58..3f9ee4507 100644 --- a/docs/source/tutorials/low-latency.rst +++ b/docs/source/tutorials/low-latency.rst @@ -28,6 +28,13 @@ disabled for these tests (:code:`pipeline.setXLinkChunkSize(0)`). For an example - 150 - 2.8 Gbps - `link `__ + * - Color (isp) + - 4K + - 26 + - 26 + - 83 (Std: 3.6) + - 2.6 Gbps + - / * - Mono - 720P/800P - 120 From 93a1dedf444e62a2080e548425d3ba74d01e4007 Mon Sep 17 00:00:00 2001 From: Martin Peterlin Date: Mon, 25 Sep 2023 20:59:58 +0200 Subject: [PATCH 76/87] Updated FW & XLink with backward compatible timestamping --- depthai-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/depthai-core b/depthai-core index b92854229..a441ae287 160000 --- a/depthai-core +++ b/depthai-core @@ -1 +1 @@ -Subproject commit b928542294770e42c1561202cefa0e8a0afc338c +Subproject commit a441ae2877a331a95530403998b0eb224a75f51e From 13bcc978955abae72bb1bdddcc13c78d5580e331 Mon Sep 17 00:00:00 2001 From: alex-luxonis Date: Thu, 28 Sep 2023 06:25:13 +0300 Subject: [PATCH 77/87] FW: fixes for AR0234 fsync, AR0234 AE max exposure: 16->33 ms, more robust camera handling (especially in sync modes) --- depthai-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/depthai-core b/depthai-core index a441ae287..66ec09080 160000 --- a/depthai-core +++ b/depthai-core @@ -1 +1 @@ -Subproject commit a441ae2877a331a95530403998b0eb224a75f51e +Subproject commit 66ec090801ff02b94c8289c9462a78c7eb87eeea From eca3a747f519341fe285004e3db1a948fd8f9126 Mon Sep 17 00:00:00 2001 From: alex-luxonis Date: Thu, 28 Sep 2023 06:27:46 +0300 Subject: [PATCH 78/87] cam_test: better display for camera metadata (`/` key) --- utilities/cam_test.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/utilities/cam_test.py b/utilities/cam_test.py index 6ee7a9546..eabd9cfb5 100755 --- a/utilities/cam_test.py +++ b/utilities/cam_test.py @@ -407,6 +407,9 @@ def exit_cleanly(signum, frame): txt += f"ISO: {pkt.getSensitivity():4}, " txt += f"Lens pos: {pkt.getLensPosition():3}, " txt += f"Color temp: {pkt.getColorTemperature()} K" + if needs_newline: + print() + needs_newline = False print(txt) capture = c in capture_list if capture: @@ -452,7 +455,7 @@ def exit_cleanly(signum, frame): print("\rFPS:", *["{:6.2f}|{:6.2f}".format(fps_host[c].get(), fps_capt[c].get()) for c in cam_list], end=' ', flush=True) - if show: print() + needs_newline = True key = cv2.waitKey(1) if key == ord('q'): @@ -626,3 +629,5 @@ def exit_cleanly(signum, frame): tofConfig.depthParams.minimumAmplitude = amp_min tofCfgQueue.send(tofConfig) controlQueue.send(ctrl) + + print() From c1cb1dda0ac9b0e2eb0f3a001eb91223f35cdcba Mon Sep 17 00:00:00 2001 From: saching13 Date: Thu, 28 Sep 2023 22:59:35 -0700 Subject: [PATCH 79/87] fixed the order of translation multiplication --- depthai-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/depthai-core b/depthai-core index 66ec09080..143defcfc 160000 --- a/depthai-core +++ b/depthai-core @@ -1 +1 @@ -Subproject commit 66ec090801ff02b94c8289c9462a78c7eb87eeea +Subproject commit 143defcfc1ee163bfeb8fb5dca1d329ab6bcfed6 From e3533c6c55a03bb20178ec202e81b6666d459e6b Mon Sep 17 00:00:00 2001 From: saching13 Date: Thu, 28 Sep 2023 23:25:15 -0700 Subject: [PATCH 80/87] update core --- depthai-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/depthai-core b/depthai-core index 143defcfc..02b1be7b6 160000 --- a/depthai-core +++ b/depthai-core @@ -1 +1 @@ -Subproject commit 143defcfc1ee163bfeb8fb5dca1d329ab6bcfed6 +Subproject commit 02b1be7b699f6f36060bd48b6adb970d212b3e22 From 78591b7214c316959440651b4884e4fc0aa1d902 Mon Sep 17 00:00:00 2001 From: Martin Peterlin Date: Fri, 29 Sep 2023 14:46:56 +0200 Subject: [PATCH 81/87] [FW] deviceName fixes and updated for EepromData changes --- depthai-core | 2 +- examples/ColorCamera/rgb_preview.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/depthai-core b/depthai-core index 66ec09080..b754f590a 160000 --- a/depthai-core +++ b/depthai-core @@ -1 +1 @@ -Subproject commit 66ec090801ff02b94c8289c9462a78c7eb87eeea +Subproject commit b754f590a8a8af1444701b46221a660b6d23844b diff --git a/examples/ColorCamera/rgb_preview.py b/examples/ColorCamera/rgb_preview.py index 0cdc49b63..332f3e1de 100755 --- a/examples/ColorCamera/rgb_preview.py +++ b/examples/ColorCamera/rgb_preview.py @@ -30,7 +30,7 @@ if device.getBootloaderVersion() is not None: print('Bootloader version:', device.getBootloaderVersion()) # Device name - print('Device name:', device.getDeviceName()) + print('Device name:', device.getDeviceName(), ' Product name:', device.getProductName()) # Output queue will be used to get the rgb frames from the output defined above qRgb = device.getOutputQueue(name="rgb", maxSize=4, blocking=False) From 116204e95e60b840bb2612f5efb70d418bd3181c Mon Sep 17 00:00:00 2001 From: Martin Peterlin Date: Fri, 29 Sep 2023 19:00:49 +0200 Subject: [PATCH 82/87] Bump to v2.23.0.0 --- depthai-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/depthai-core b/depthai-core index b754f590a..ff0617ba1 160000 --- a/depthai-core +++ b/depthai-core @@ -1 +1 @@ -Subproject commit b754f590a8a8af1444701b46221a660b6d23844b +Subproject commit ff0617ba10e91cc42272afa7bf2a4c9f289c019e From 9d3ccd935fbe3ccdc4459ec6549a38064e4163a1 Mon Sep 17 00:00:00 2001 From: Martin Peterlin Date: Sat, 30 Sep 2023 23:58:29 +0200 Subject: [PATCH 83/87] Fixed verticalCameraSocket bindings --- src/pipeline/CommonBindings.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipeline/CommonBindings.cpp b/src/pipeline/CommonBindings.cpp index 8304d12ce..2b52843af 100644 --- a/src/pipeline/CommonBindings.cpp +++ b/src/pipeline/CommonBindings.cpp @@ -262,7 +262,6 @@ void CommonBindings::bind(pybind11::module& m, void* pCallstack){ .def_readwrite("rectifiedRotationRight", &StereoRectification::rectifiedRotationRight) .def_readwrite("leftCameraSocket", &StereoRectification::leftCameraSocket) .def_readwrite("rightCameraSocket", &StereoRectification::rightCameraSocket) - .def_readwrite("verticalCameraSocket", &StereoRectification::verticalCameraSocket) ; // Extrinsics @@ -307,6 +306,7 @@ void CommonBindings::bind(pybind11::module& m, void* pCallstack){ .def_readwrite("housingExtrinsics", &EepromData::housingExtrinsics) .def_readwrite("stereoUseSpecTranslation", &EepromData::stereoUseSpecTranslation) .def_readwrite("stereoEnableDistortionCorrection", &EepromData::stereoEnableDistortionCorrection) + .def_readwrite("verticalCameraSocket", &EepromData::verticalCameraSocket) ; // UsbSpeed usbSpeed From a4c0d1a61c359f57b15fecc9e5081d63d14162b5 Mon Sep 17 00:00:00 2001 From: Erol444 Date: Mon, 2 Oct 2023 13:32:13 +0200 Subject: [PATCH 84/87] Update typo --- docs/source/tutorials/low-latency.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/tutorials/low-latency.rst b/docs/source/tutorials/low-latency.rst index 3f9ee4507..484e9aa61 100644 --- a/docs/source/tutorials/low-latency.rst +++ b/docs/source/tutorials/low-latency.rst @@ -100,7 +100,7 @@ without any switches or routers in between. Power was supplied via M8 connector. - 8 Std: 0.45 - 225 Mbps -We set lower FPS for the POE measurements due to bandwidth constraints. For example, 4K 8 FPS was 150ms latency, while +We set lower FPS for the POE measurements due to bandwidth constraints. For example, 4K 8 FPS had 150ms latency, while 4K 10FPS had 530ms latency, as link was saturated. - **Latency** is measured time between frame timestamp (:code:`imgFrame.getTimestamp()`) and host timestamp when the frame is received (:code:`dai.Clock.now()`). From 82677440ff840579162511c89e5301aef485504c Mon Sep 17 00:00:00 2001 From: Matevz Morato Date: Tue, 28 Nov 2023 14:32:35 +0000 Subject: [PATCH 85/87] [RVC3 FW] Add getMxId implementation --- depthai-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/depthai-core b/depthai-core index 636e5ac7e..05c8d594d 160000 --- a/depthai-core +++ b/depthai-core @@ -1 +1 @@ -Subproject commit 636e5ac7e0d10ac083bc72a8630f7ae543c8c05d +Subproject commit 05c8d594d9756c348fcd3d04ba9b7858b4b79e02 From 4ee707e88dec25b2763ecf2693828f91d9722e1f Mon Sep 17 00:00:00 2001 From: Matevz Morato Date: Thu, 7 Dec 2023 17:43:51 +0000 Subject: [PATCH 86/87] [RVC3 FW] Fix for Feature tracker --- depthai-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/depthai-core b/depthai-core index 05c8d594d..aee4414a9 160000 --- a/depthai-core +++ b/depthai-core @@ -1 +1 @@ -Subproject commit 05c8d594d9756c348fcd3d04ba9b7858b4b79e02 +Subproject commit aee4414a9944ea0184f58dc1aed21d9f8492bab6 From dca4106959b48c6c03aa49434fab54227136fda1 Mon Sep 17 00:00:00 2001 From: Matevz Morato Date: Thu, 14 Dec 2023 14:12:35 +0000 Subject: [PATCH 87/87] Change the sync node to work on buffers --- depthai-core | 2 +- examples/Sync/sync_left_right.py | 43 ++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 examples/Sync/sync_left_right.py diff --git a/depthai-core b/depthai-core index df44897e1..7890a60fc 160000 --- a/depthai-core +++ b/depthai-core @@ -1 +1 @@ -Subproject commit df44897e10d2b897ad3821e43b98952d4b07eab8 +Subproject commit 7890a60fcce1f34ab8039d5329bb4be54d1ef74d diff --git a/examples/Sync/sync_left_right.py b/examples/Sync/sync_left_right.py new file mode 100644 index 000000000..2bbd2a1ac --- /dev/null +++ b/examples/Sync/sync_left_right.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 + +import cv2 +import depthai as dai +import numpy as np + + +# Create pipeline +pipeline = dai.Pipeline() + +# Define sources and outputs +monoLeft = pipeline.create(dai.node.MonoCamera) +monoRight = pipeline.create(dai.node.MonoCamera) +xoutLeft = pipeline.create(dai.node.XLinkOut) +xoutRight = pipeline.create(dai.node.XLinkOut) +syncNode = pipeline.create(dai.node.Sync) + +xoutLeft.setStreamName("left") +xoutRight.setStreamName("right") + + +# Properties +monoLeft.setResolution(dai.MonoCameraProperties.SensorResolution.THE_720_P) +monoLeft.setBoardSocket(dai.CameraBoardSocket.LEFT) +monoRight.setResolution(dai.MonoCameraProperties.SensorResolution.THE_720_P) +monoRight.setBoardSocket(dai.CameraBoardSocket.RIGHT) + +monoLeft.out.link(syncNode.inputs["left"]) +monoRight.out.link(syncNode.inputs["right"]) +syncNode.outputs["left"].link(xoutLeft.input) +syncNode.outputs["right"].link(xoutRight.input) + +with dai.Device(pipeline) as device: + qLeft = device.getOutputQueue(name="left", maxSize=4, blocking=False) + qRight = device.getOutputQueue(name="right", maxSize=4, blocking=False) + while True: + inLeft = qLeft.get() + inRight = qRight.get() + + cv2.imshow("left", inLeft.getCvFrame()) + cv2.imshow("right", inRight.getCvFrame()) + if cv2.waitKey(1) == ord('q'): + break \ No newline at end of file