From 3f158b01a06aea87617c562e75430bf22f4840c0 Mon Sep 17 00:00:00 2001 From: olga Date: Mon, 4 Dec 2023 15:47:20 +0100 Subject: [PATCH 1/6] Loading all available data from folder --- src/viewer/src/App.tsx | 72 ++++++--------- .../{ => components}/CostMatrixComponent.tsx | 59 ++++++++++--- src/viewer/src/components/DataLoader.tsx | 74 ++++++++++++++++ .../src/{ => components}/ImageCostMatrix.tsx | 2 +- .../src/{ => components}/ImagesLoader.css | 0 .../ImagesPreview.tsx} | 66 +++++++++----- .../InteractiveCostMatrix.tsx | 4 +- .../src/{ => components}/ProtoLoader.tsx | 0 src/viewer/src/matchingResult.tsx | 17 ---- src/viewer/src/{ => resources}/costMatrix.ts | 0 src/viewer/src/resources/readers.ts | 87 +++++++++++++++++++ 11 files changed, 280 insertions(+), 101 deletions(-) rename src/viewer/src/{ => components}/CostMatrixComponent.tsx (71%) create mode 100644 src/viewer/src/components/DataLoader.tsx rename src/viewer/src/{ => components}/ImageCostMatrix.tsx (99%) rename src/viewer/src/{ => components}/ImagesLoader.css (100%) rename src/viewer/src/{ImagesLoader.tsx => components/ImagesPreview.tsx} (73%) rename src/viewer/src/{ => components}/InteractiveCostMatrix.tsx (97%) rename src/viewer/src/{ => components}/ProtoLoader.tsx (100%) delete mode 100644 src/viewer/src/matchingResult.tsx rename src/viewer/src/{ => resources}/costMatrix.ts (100%) create mode 100644 src/viewer/src/resources/readers.ts diff --git a/src/viewer/src/App.tsx b/src/viewer/src/App.tsx index 64a6879..07258af 100644 --- a/src/viewer/src/App.tsx +++ b/src/viewer/src/App.tsx @@ -1,36 +1,22 @@ // Created by O. Vysotska in 2023 -import { useState, useEffect } from "react"; +import { useState } from "react"; import "./App.css"; -import { ProtoLoader, MessageType } from "./ProtoLoader"; -import { CostMatrix, CostMatrixElement } from "./costMatrix"; -import { ImagesLoader } from "./ImagesLoader"; -import { - MatchingResultElement, - readMatchingResultFromProto, -} from "./matchingResult"; -import CostMatrixComponent from "./CostMatrixComponent"; +import { CostMatrixElement } from "./resources/costMatrix"; +import { ImagesPreview } from "./components/ImagesPreview"; +import CostMatrixComponent from "./components/CostMatrixComponent"; + +import DataLoader from "./components/DataLoader"; function App() { - const [costMatrixProto, setCostMatrixProto] = useState(null); - const [costMatrix, setCostMatrix] = useState(); - const [matchingResultProto, setMatchingResultProto] = useState(null); - const [matchingResult, setMatchingResult] = - useState(); + const [costMatrixProtoFile, setCostMatrixProtoFile] = useState(); + const [matchingResultProtoFile, setMatchingResultProtoFile] = + useState(); + const [queryImageFiles, setQueryImageFiles] = useState(); + const [referenceImageFiles, setReferenceImageFiles] = useState(); + const [selectedCostMatrixElement, setSelectedCostMatrixElement] = useState(); - useEffect(() => { - if (costMatrixProto != null) { - console.log("Cost Matrix proto is loaded", costMatrixProto); - setCostMatrix(new CostMatrix(costMatrixProto)); - } - }, [costMatrixProto]); - - useEffect(() => { - console.log("Matching result proto changed", matchingResultProto); - setMatchingResult(readMatchingResultFromProto(matchingResultProto)); - }, [matchingResultProto]); - return (

Cost Matrix Viewer

+
-
-
- - -
-
- {costMatrix && ( + {costMatrixProtoFile && ( )} @@ -77,14 +57,16 @@ function App() { }} >
-
-
diff --git a/src/viewer/src/CostMatrixComponent.tsx b/src/viewer/src/components/CostMatrixComponent.tsx similarity index 71% rename from src/viewer/src/CostMatrixComponent.tsx rename to src/viewer/src/components/CostMatrixComponent.tsx index 0ddb24c..b5166bf 100644 --- a/src/viewer/src/CostMatrixComponent.tsx +++ b/src/viewer/src/components/CostMatrixComponent.tsx @@ -1,12 +1,18 @@ // Created by O. Vysotska in 2023 import { useState, useEffect } from "react"; -import { CostMatrix, CostMatrixElement } from "./costMatrix"; +import { CostMatrix, CostMatrixElement } from "../resources/costMatrix"; import { ImageCostMatrix, ZoomBlockParams } from "./ImageCostMatrix"; -import { MatchingResultElement } from "./matchingResult"; import InteractiveCostMatrix from "./InteractiveCostMatrix"; import { FormGroup, FormControlLabel, Switch } from "@mui/material"; +import { + readMatchingResultFromProto, + readProtoFromFile, + MatchingResultElement, + ProtoMessageType, +} from "../resources/readers"; + function getMatchingResultInZoomBlock( results: MatchingResultElement[], zoomParams: ZoomBlockParams @@ -21,12 +27,13 @@ function getMatchingResultInZoomBlock( } type CostMatrixProps = { - costMatrix: CostMatrix; - matchingResult?: MatchingResultElement[]; + costMatrixProtoFile: File; + matchingResultProtoFile?: File; setSelectedCostMatrixElement: (element: CostMatrixElement) => void; }; function CostMatrixComponent(props: CostMatrixProps): React.ReactElement { + const [costMatrix, setCostMatrix] = useState(); const [image, setImage] = useState(); const [matchingResult, setMatchingResult] = useState(); @@ -37,6 +44,21 @@ function CostMatrixComponent(props: CostMatrixProps): React.ReactElement { const [selectedElement, setSelectedElement] = useState(); const { setSelectedCostMatrixElement } = props; + + // Read costMatrix from proto file. + useEffect(() => { + if (props.costMatrixProtoFile == null) { + return; + } + readProtoFromFile(props.costMatrixProtoFile, ProtoMessageType.CostMatrix) + .then((costMatrixProto) => { + setCostMatrix(new CostMatrix(costMatrixProto)); + }) + .catch((e) => { + console.log("Couldn't read file", props.costMatrixProtoFile); + }); + }, [props.costMatrixProtoFile]); + useEffect(() => { if (selectedElement == null) { return; @@ -45,29 +67,42 @@ function CostMatrixComponent(props: CostMatrixProps): React.ReactElement { }, [selectedElement, setSelectedCostMatrixElement]); useEffect(() => { - if (props.costMatrix != null) { - props.costMatrix.createImage().then((result) => { + if (costMatrix != null) { + costMatrix.createImage().then((result) => { setImage(result); }); } - }, [props.costMatrix]); + }, [costMatrix]); + // Read Matching result proto file useEffect(() => { - setMatchingResult(props.matchingResult); - }, [props.matchingResult]); + if (props.matchingResultProtoFile == null) { + return; + } + readProtoFromFile( + props.matchingResultProtoFile, + ProtoMessageType.MatchingResult + ) + .then((matchingResultProto) => { + setMatchingResult(readMatchingResultFromProto(matchingResultProto)); + }) + .catch((e) => { + console.log("Couldn't read file", props.matchingResultProtoFile); + }); + }, [props.matchingResultProtoFile]); useEffect(() => { - if (zoomParams == null || props.costMatrix == null) { + if (zoomParams == null || costMatrix == null) { return; } setZoomedCostMatrix( - props.costMatrix.getSubMatrix( + costMatrix.getSubMatrix( zoomParams.topLeftX, zoomParams.topLeftY, zoomParams.windowHeightPx ) ); - }, [zoomParams, props.costMatrix]); + }, [zoomParams, costMatrix]); function showMatchingResult(event: any) { if (event.target.checked) { diff --git a/src/viewer/src/components/DataLoader.tsx b/src/viewer/src/components/DataLoader.tsx new file mode 100644 index 0000000..0b39c3c --- /dev/null +++ b/src/viewer/src/components/DataLoader.tsx @@ -0,0 +1,74 @@ +type DataLoaderProps = { + setMatchingResultProtoFile: (file: File) => void; + setCostMatrixProtoFile: (file: File) => void; + setQueryImageFiles: (files: File[]) => void; + setReferenceImageFiles: (file: File[]) => void; +}; + +function DataLoader(props: DataLoaderProps): React.ReactElement { + async function onChange(event: React.ChangeEvent) { + const fileList = event.target.files; + if (fileList == null) { + return; + } + const files = Array.from(fileList); + + const costMatrixProtoFile = files.find((file) => { + return file.webkitRelativePath.split("/")[1].endsWith(".CostMatrix.pb"); + }); + if (costMatrixProtoFile == null) { + console.warn("CostMatrix proto file was not found"); + } else { + props.setCostMatrixProtoFile(costMatrixProtoFile); + } + + const matchingResultProtoFile = files.find((file) => { + return file.webkitRelativePath + .split("/")[1] + .endsWith(".MatchingResult.pb"); + }); + + if (matchingResultProtoFile == null) { + console.warn("MatchingResult proto file was not found"); + } else { + props.setMatchingResultProtoFile(matchingResultProtoFile); + } + + const queryImageFiles = files.filter((file) => { + console.log(file.webkitRelativePath.split("/")[1]); + return file.webkitRelativePath.split("/")[1] === "query_images"; + }); + if (queryImageFiles.length === 0) { + console.warn("Query images was not found"); + } else { + props.setQueryImageFiles(queryImageFiles); + } + + const referenceImageFiles = files.filter((file) => { + return file.webkitRelativePath.split("/")[1] === "reference_images"; + }); + if (referenceImageFiles.length === 0) { + console.warn("Reference images was not found"); + } else { + props.setReferenceImageFiles(referenceImageFiles); + } + } + + return ( +
+ + +
+ ); +} + +export default DataLoader; diff --git a/src/viewer/src/ImageCostMatrix.tsx b/src/viewer/src/components/ImageCostMatrix.tsx similarity index 99% rename from src/viewer/src/ImageCostMatrix.tsx rename to src/viewer/src/components/ImageCostMatrix.tsx index b54e01f..9845b0e 100644 --- a/src/viewer/src/ImageCostMatrix.tsx +++ b/src/viewer/src/components/ImageCostMatrix.tsx @@ -1,6 +1,6 @@ import React, { forwardRef } from "react"; import { useState, useRef, useEffect } from "react"; -import { MatchingResultElement } from "./matchingResult"; +import { MatchingResultElement } from "../resources/readers"; const kZoomWindowPx = 30; diff --git a/src/viewer/src/ImagesLoader.css b/src/viewer/src/components/ImagesLoader.css similarity index 100% rename from src/viewer/src/ImagesLoader.css rename to src/viewer/src/components/ImagesLoader.css diff --git a/src/viewer/src/ImagesLoader.tsx b/src/viewer/src/components/ImagesPreview.tsx similarity index 73% rename from src/viewer/src/ImagesLoader.tsx rename to src/viewer/src/components/ImagesPreview.tsx index 3bc2ffe..d2e4e8e 100644 --- a/src/viewer/src/ImagesLoader.tsx +++ b/src/viewer/src/components/ImagesPreview.tsx @@ -23,28 +23,19 @@ type ImageData = { base64Encoding?: string; }; -type ImageLoaderProps = { - imageType: string; +type ImagePreviewProps = { + imageFiles?: File[]; + imageSource: string; showImageId?: number; }; -function ImagesLoader(props: ImageLoaderProps) { +function ImagesPreview(props: ImagePreviewProps) { const [images, setImages] = useState(); const [currentImageId, setCurrentImageId] = useState(0); - useEffect(() => { - if (images == null || props.showImageId == null) { - return; - } - if (props.showImageId < 0 || props.showImageId >= images.length) { - return; - } - setCurrentImageId(props.showImageId); - }, [props.showImageId, images]); + const { imageFiles, showImageId } = props; - function onChange(event: React.ChangeEvent) { - event.preventDefault(); - let files = event.target.files; + function loadImages(files?: File[]) { if (files == null) { return; } @@ -66,12 +57,37 @@ function ImagesLoader(props: ImageLoaderProps) { setImages(imageList); }) .catch((error) => { - console.log("Image was not loaded"); + console.log("Image was not loaded", error); }); } }); } + useEffect(() => { + // Loads from props + loadImages(imageFiles); + }, [imageFiles]); + + useEffect(() => { + if (images == null || showImageId == null) { + return; + } + if (showImageId < 0 || showImageId >= images.length) { + return; + } + setCurrentImageId(showImageId); + }, [showImageId, images]); + + function onChange(event: React.ChangeEvent) { + // Loads from input + event.preventDefault(); + let files = event.target.files; + if (files == null) { + return; + } + loadImages(Array.from(files)); + } + function handleNextClick() { if (images == null) { return; @@ -92,12 +108,14 @@ function ImagesLoader(props: ImageLoaderProps) { textAlign: "center", }} > -

{props.imageType} images

- -
- - -
+

{props.imageSource} images

+ + {!props.imageFiles && ( +
+ + +
+ )} {images && images.length > currentImageId && images[currentImageId] !== undefined && ( @@ -118,5 +136,5 @@ function ImagesLoader(props: ImageLoaderProps) { ); } -export { ImagesLoader }; -export type { ImageLoaderProps }; +export { ImagesPreview }; +export type { ImagePreviewProps }; diff --git a/src/viewer/src/InteractiveCostMatrix.tsx b/src/viewer/src/components/InteractiveCostMatrix.tsx similarity index 97% rename from src/viewer/src/InteractiveCostMatrix.tsx rename to src/viewer/src/components/InteractiveCostMatrix.tsx index c7183b9..9242142 100644 --- a/src/viewer/src/InteractiveCostMatrix.tsx +++ b/src/viewer/src/components/InteractiveCostMatrix.tsx @@ -1,10 +1,10 @@ import React from "react"; import { useState, useRef, useEffect, useCallback } from "react"; -import { CostMatrix, CostMatrixElement } from "./costMatrix"; +import { CostMatrix, CostMatrixElement } from "../resources/costMatrix"; import { ZoomBlockParams } from "./ImageCostMatrix"; import * as d3 from "d3"; -import { MatchingResultElement } from "./matchingResult"; +import { MatchingResultElement } from "../resources/readers"; type TooltipProps = { opacity: number; diff --git a/src/viewer/src/ProtoLoader.tsx b/src/viewer/src/components/ProtoLoader.tsx similarity index 100% rename from src/viewer/src/ProtoLoader.tsx rename to src/viewer/src/components/ProtoLoader.tsx diff --git a/src/viewer/src/matchingResult.tsx b/src/viewer/src/matchingResult.tsx deleted file mode 100644 index e168998..0000000 --- a/src/viewer/src/matchingResult.tsx +++ /dev/null @@ -1,17 +0,0 @@ -type MatchingResultElement = { - queryId: number; - refId: number; - real: boolean; -}; - -function readMatchingResultFromProto( - matchingResultProto?: any -): MatchingResultElement[] | undefined { - if (matchingResultProto == null) { - return undefined; - } - return matchingResultProto.matches; -} - -export { type MatchingResultElement }; -export { readMatchingResultFromProto }; diff --git a/src/viewer/src/costMatrix.ts b/src/viewer/src/resources/costMatrix.ts similarity index 100% rename from src/viewer/src/costMatrix.ts rename to src/viewer/src/resources/costMatrix.ts diff --git a/src/viewer/src/resources/readers.ts b/src/viewer/src/resources/readers.ts new file mode 100644 index 0000000..7651931 --- /dev/null +++ b/src/viewer/src/resources/readers.ts @@ -0,0 +1,87 @@ +function readFileAsync(file: Blob) { + return new Promise((resolve, reject) => { + let reader = new FileReader(); + reader.onload = () => { + resolve(reader.result as ArrayBuffer); + }; + reader.onerror = reject; + reader.readAsArrayBuffer(file); + }); +} + +function readImageAsync(file: Blob) { + return new Promise((resolve, reject) => { + let reader = new FileReader(); + reader.onload = () => { + resolve(reader.result as string); + }; + reader.onerror = reject; + reader.readAsDataURL(file); + }); +} + +enum ProtoMessageType { + CostMatrix = "CostMatrix", + MatchingResult = "MatchingResult", +} + +function readProtoFromBuffer(buffer: Uint8Array, protoMessageType: string) { + return new Promise((resolve, reject) => { + const protobuf = require("protobufjs"); + protobuf + .load("localization_protos.proto") + .then(function (root: any) { + // Get the message type from the root object + const message = root.lookupType(protoMessageType); + const decodedMessage = message.decode(buffer); + console.log("Message", decodedMessage); + resolve(decodedMessage); + }) + .catch((error: any) => { + console.log("ERROR, proto couldn't be loaded", error); + reject(); + }); + }); +} + +async function readProtoFromFile(file: File, messageType: ProtoMessageType) { + try { + let contentBuffer = await readFileAsync(file); + if (contentBuffer == null) { + throw new Error("Empty content"); + } + let protoMessage = await readProtoFromBuffer( + new Uint8Array(contentBuffer), + "image_sequence_localizer." + messageType.toString() + ); + console.log("Read proto message", protoMessage); + return protoMessage; + } catch (err) { + console.log("File was not processed", err); + } +} + +type MatchingResultElement = { + queryId: number; + refId: number; + real: boolean; +}; + +function readMatchingResultFromProto( + matchingResultProto?: any +): MatchingResultElement[] | undefined { + if (matchingResultProto == null) { + return undefined; + } + return matchingResultProto.matches; +} + +export { + readFileAsync, + readImageAsync, + readProtoFromFile, + readMatchingResultFromProto, + ProtoMessageType, +}; + +export type { MatchingResultElement }; From cb1e2a9630173c427062cf961b3b31ac75cb5509 Mon Sep 17 00:00:00 2001 From: olga Date: Tue, 5 Dec 2023 17:10:00 +0100 Subject: [PATCH 2/6] upload folder instead of individual files --- src/viewer/src/components/DataLoader.tsx | 5 +++- src/viewer/src/components/ImagesPreview.tsx | 31 ++++++++++++++------- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/src/viewer/src/components/DataLoader.tsx b/src/viewer/src/components/DataLoader.tsx index 0b39c3c..8d3b752 100644 --- a/src/viewer/src/components/DataLoader.tsx +++ b/src/viewer/src/components/DataLoader.tsx @@ -1,3 +1,5 @@ +import Alert from "@mui/material/Alert"; + type DataLoaderProps = { setMatchingResultProtoFile: (file: File) => void; setCostMatrixProtoFile: (file: File) => void; @@ -39,7 +41,7 @@ function DataLoader(props: DataLoaderProps): React.ReactElement { return file.webkitRelativePath.split("/")[1] === "query_images"; }); if (queryImageFiles.length === 0) { - console.warn("Query images was not found"); + props.setQueryImageFiles([]); } else { props.setQueryImageFiles(queryImageFiles); } @@ -49,6 +51,7 @@ function DataLoader(props: DataLoaderProps): React.ReactElement { }); if (referenceImageFiles.length === 0) { console.warn("Reference images was not found"); + props.setReferenceImageFiles([]); } else { props.setReferenceImageFiles(referenceImageFiles); } diff --git a/src/viewer/src/components/ImagesPreview.tsx b/src/viewer/src/components/ImagesPreview.tsx index d2e4e8e..35748e4 100644 --- a/src/viewer/src/components/ImagesPreview.tsx +++ b/src/viewer/src/components/ImagesPreview.tsx @@ -1,10 +1,9 @@ import { useState, useEffect } from "react"; -import "./ImagesLoader.css" +import "./ImagesLoader.css"; -import ArrowForwardIosRoundedIcon from '@mui/icons-material/ArrowForwardIosRounded'; -import ArrowBackIosRoundedIcon from '@mui/icons-material/ArrowBackIosRounded'; -import { Input } from '@mui/material'; +import ArrowForwardIosRoundedIcon from "@mui/icons-material/ArrowForwardIosRounded"; +import ArrowBackIosRoundedIcon from "@mui/icons-material/ArrowBackIosRounded"; function readImageAsync(file: Blob) { return new Promise((resolve, reject) => { @@ -39,6 +38,9 @@ function ImagesPreview(props: ImagePreviewProps) { if (files == null) { return; } + if (files.length === 0) { + setImages(undefined); + } const sortedFiles = Array.from(files).sort((a, b) => { return a.name.localeCompare(b.name); @@ -110,7 +112,7 @@ function ImagesPreview(props: ImagePreviewProps) { >

{props.imageSource} images

- {!props.imageFiles && ( + {!images && (
@@ -120,18 +122,27 @@ function ImagesPreview(props: ImagePreviewProps) { images.length > currentImageId && images[currentImageId] !== undefined && (
- preview + preview

id: {images[currentImageId].id}; filename:{" "} {images[currentImageId].fileName}

- - + +
)} -
-
+
); } From 311b07128f40a0a0f5e105f73ec8b97df9564d81 Mon Sep 17 00:00:00 2001 From: olga Date: Wed, 6 Dec 2023 15:02:37 +0100 Subject: [PATCH 3/6] added command to create hard link for images. Added functionality to use context for selected element --- src/python/run_matching.py | 29 +++++++ src/viewer/src/App.tsx | 81 +++++++----------- .../src/components/CostMatrixComponent.tsx | 14 +-- src/viewer/src/components/DataLoader.tsx | 16 ++-- .../{ImagesLoader.css => ImageCarousel.css} | 3 + .../{ImagesPreview.tsx => ImageCarousel.tsx} | 10 +-- .../src/components/ImagePreviewComponent.tsx | 48 +++++++++++ .../src/components/InteractiveCostMatrix.tsx | 23 +++-- src/viewer/src/components/ProtoLoader.tsx | 85 ------------------- src/viewer/src/context/ElementContext.tsx | 42 +++++++++ 10 files changed, 180 insertions(+), 171 deletions(-) rename src/viewer/src/components/{ImagesLoader.css => ImageCarousel.css} (88%) rename src/viewer/src/components/{ImagesPreview.tsx => ImageCarousel.tsx} (95%) create mode 100644 src/viewer/src/components/ImagePreviewComponent.tsx delete mode 100644 src/viewer/src/components/ProtoLoader.tsx create mode 100644 src/viewer/src/context/ElementContext.tsx diff --git a/src/python/run_matching.py b/src/python/run_matching.py index acbfc7f..f9320c7 100644 --- a/src/python/run_matching.py +++ b/src/python/run_matching.py @@ -35,6 +35,11 @@ def parseParams(): action="store_true", help="Creates and writes the pair of matching images.", ) + parser.add_argument( + "--link_images", + action="store_true", + help="Creates hard link for images in the result folder.", + ) return parser.parse_args() @@ -152,6 +157,26 @@ def runMatchingResultVisualization(config, output_dir): os.system(command) +def linkImages(image_dir, output_folder): + if output_folder.exists(): + print( + "Output {output_folder} exists. Skipping...".format( + output_folder=output_folder + ) + ) + return + output_folder.mkdir() + images = image_dir.glob("*") + for image in images: + image_link_name = output_folder / image.name + command = "ln {image} {image_link_name}".format( + image=image, image_link_name=image_link_name + ) + print(command) + os.system(command) + print("Linked images from", image_dir) + + def main(): args = parseParams() @@ -160,6 +185,10 @@ def main(): else: args.output_dir.mkdir() + if args.link_images: + linkImages(args.query_images, args.output_dir / "query_images") + linkImages(args.reference_images, args.output_dir / "reference_images") + # Compute query features. query_features_dir = args.output_dir / "query_features" computeFeatures(args.query_images, query_features_dir, args.dataset_name) diff --git a/src/viewer/src/App.tsx b/src/viewer/src/App.tsx index 07258af..d6e6c86 100644 --- a/src/viewer/src/App.tsx +++ b/src/viewer/src/App.tsx @@ -1,12 +1,13 @@ // Created by O. Vysotska in 2023 import { useState } from "react"; import "./App.css"; -import { CostMatrixElement } from "./resources/costMatrix"; -import { ImagesPreview } from "./components/ImagesPreview"; -import CostMatrixComponent from "./components/CostMatrixComponent"; +import CostMatrixComponent from "./components/CostMatrixComponent"; +import { ImagePreviewComponent } from "./components/ImagePreviewComponent"; import DataLoader from "./components/DataLoader"; +import { ElementProvider } from "./context/ElementContext"; + function App() { const [costMatrixProtoFile, setCostMatrixProtoFile] = useState(); const [matchingResultProtoFile, setMatchingResultProtoFile] = @@ -14,65 +15,43 @@ function App() { const [queryImageFiles, setQueryImageFiles] = useState(); const [referenceImageFiles, setReferenceImageFiles] = useState(); - const [selectedCostMatrixElement, setSelectedCostMatrixElement] = - useState(); - return ( -
-
-

Cost Matrix Viewer

- -
- {costMatrixProtoFile && ( - - )} -
- + +
-
- -
-
- +

Cost Matrix Viewer

+ +
+ {costMatrixProtoFile && ( + + )}
+
-
+ ); } diff --git a/src/viewer/src/components/CostMatrixComponent.tsx b/src/viewer/src/components/CostMatrixComponent.tsx index b5166bf..62dcc92 100644 --- a/src/viewer/src/components/CostMatrixComponent.tsx +++ b/src/viewer/src/components/CostMatrixComponent.tsx @@ -1,6 +1,6 @@ // Created by O. Vysotska in 2023 import { useState, useEffect } from "react"; -import { CostMatrix, CostMatrixElement } from "../resources/costMatrix"; +import { CostMatrix } from "../resources/costMatrix"; import { ImageCostMatrix, ZoomBlockParams } from "./ImageCostMatrix"; import InteractiveCostMatrix from "./InteractiveCostMatrix"; @@ -29,7 +29,6 @@ function getMatchingResultInZoomBlock( type CostMatrixProps = { costMatrixProtoFile: File; matchingResultProtoFile?: File; - setSelectedCostMatrixElement: (element: CostMatrixElement) => void; }; function CostMatrixComponent(props: CostMatrixProps): React.ReactElement { @@ -41,9 +40,6 @@ function CostMatrixComponent(props: CostMatrixProps): React.ReactElement { const [zoomedCostMatrix, setZoomedCostMatrix] = useState(); const [matchingResultVisible, setMatchingResultVisible] = useState(false); - const [selectedElement, setSelectedElement] = useState(); - - const { setSelectedCostMatrixElement } = props; // Read costMatrix from proto file. useEffect(() => { @@ -59,13 +55,6 @@ function CostMatrixComponent(props: CostMatrixProps): React.ReactElement { }); }, [props.costMatrixProtoFile]); - useEffect(() => { - if (selectedElement == null) { - return; - } - setSelectedCostMatrixElement(selectedElement); - }, [selectedElement, setSelectedCostMatrixElement]); - useEffect(() => { if (costMatrix != null) { costMatrix.createImage().then((result) => { @@ -162,7 +151,6 @@ function CostMatrixComponent(props: CostMatrixProps): React.ReactElement { void; setCostMatrixProtoFile: (file: File) => void; @@ -37,10 +35,11 @@ function DataLoader(props: DataLoaderProps): React.ReactElement { } const queryImageFiles = files.filter((file) => { - console.log(file.webkitRelativePath.split("/")[1]); return file.webkitRelativePath.split("/")[1] === "query_images"; }); + console.log("Query images", queryImageFiles); if (queryImageFiles.length === 0) { + console.warn("Query images are not found"); props.setQueryImageFiles([]); } else { props.setQueryImageFiles(queryImageFiles); @@ -50,7 +49,7 @@ function DataLoader(props: DataLoaderProps): React.ReactElement { return file.webkitRelativePath.split("/")[1] === "reference_images"; }); if (referenceImageFiles.length === 0) { - console.warn("Reference images was not found"); + console.warn("Reference images are not found"); props.setReferenceImageFiles([]); } else { props.setReferenceImageFiles(referenceImageFiles); @@ -58,7 +57,14 @@ function DataLoader(props: DataLoaderProps): React.ReactElement { } return ( -
+
(); const [currentImageId, setCurrentImageId] = useState(0); @@ -147,5 +147,5 @@ function ImagesPreview(props: ImagePreviewProps) { ); } -export { ImagesPreview }; -export type { ImagePreviewProps }; +export { ImageCarousel }; +export type { ImageCarouselProps }; diff --git a/src/viewer/src/components/ImagePreviewComponent.tsx b/src/viewer/src/components/ImagePreviewComponent.tsx new file mode 100644 index 0000000..ae9d597 --- /dev/null +++ b/src/viewer/src/components/ImagePreviewComponent.tsx @@ -0,0 +1,48 @@ +import React from "react"; +import { ImageCarousel } from "./ImageCarousel"; + +import { useElementContext } from "../context/ElementContext"; + +type ImagePreviewComponentProps = { + queryImageFiles?: File[]; + referenceImageFiles?: File[]; +}; + +function ImagePreviewComponent( + props: ImagePreviewComponentProps +): React.ReactElement { + const { globalSelectedElement, setGlobalSelectedElement } = + useElementContext(); + + return ( +
+
+ +
+
+ +
+
+ ); +} + +export { ImagePreviewComponent }; diff --git a/src/viewer/src/components/InteractiveCostMatrix.tsx b/src/viewer/src/components/InteractiveCostMatrix.tsx index 9242142..000d783 100644 --- a/src/viewer/src/components/InteractiveCostMatrix.tsx +++ b/src/viewer/src/components/InteractiveCostMatrix.tsx @@ -3,6 +3,8 @@ import { useState, useRef, useEffect, useCallback } from "react"; import { CostMatrix, CostMatrixElement } from "../resources/costMatrix"; import { ZoomBlockParams } from "./ImageCostMatrix"; +import { useElementContext } from "../context/ElementContext"; + import * as d3 from "d3"; import { MatchingResultElement } from "../resources/readers"; @@ -47,7 +49,6 @@ function Tooltip(props: TooltipProps): React.ReactElement { type InteractiveCostMatrixProps = { costMatrix: CostMatrix; zoomBlock: ZoomBlockParams; - setSelectedElement: (element: CostMatrixElement) => void; matches?: MatchingResultElement[]; showMatches?: boolean; }; @@ -60,7 +61,9 @@ function InteractiveCostMatrix( const [tooltipProps, setTooltipProps] = useState(); const [matchesVisible, setMatchesVisible] = useState(false); - const [selectedPixel, setSelectedPixel] = useState(); + + const { globalSelectedElement, setGlobalSelectedElement } = + useElementContext(); const height = 500; const cellSize = height / props.costMatrix.rows; @@ -114,16 +117,12 @@ function InteractiveCostMatrix( event.target.setAttribute("opacity", 1.0); }, []); - const onCellClick = useCallback((event: any, cell: CostMatrixElement) => { - setSelectedPixel(cell); - }, []); - - const { setSelectedElement } = props; - useEffect(() => { - if (selectedPixel) { - setSelectedElement(selectedPixel); - } - }, [selectedPixel, setSelectedElement]); + const onCellClick = useCallback( + (event: any, cell: CostMatrixElement) => { + setGlobalSelectedElement(cell); + }, + [setGlobalSelectedElement] + ); useEffect(() => { // Retrieve the group for matrix elements diff --git a/src/viewer/src/components/ProtoLoader.tsx b/src/viewer/src/components/ProtoLoader.tsx deleted file mode 100644 index 602d009..0000000 --- a/src/viewer/src/components/ProtoLoader.tsx +++ /dev/null @@ -1,85 +0,0 @@ -import React, { useEffect, useState } from "react"; - -enum MessageType { - CostMatrix = "CostMatrix", - MatchingResult = "MatchingResult", -} - -function readFileAsync(file: Blob) { - return new Promise((resolve, reject) => { - let reader = new FileReader(); - reader.onload = () => { - resolve(reader.result as ArrayBuffer); - }; - reader.onerror = reject; - reader.readAsArrayBuffer(file); - }); -} - -function readProto(buffer: Uint8Array, protoMessageType: string) { - return new Promise((resolve, reject) => { - const protobuf = require("protobufjs"); - protobuf - .load("localization_protos.proto") - .then(function (root: any) { - // Get the message type from the root object - const message = root.lookupType(protoMessageType); - const decodedMessage = message.decode(buffer); - console.log("Message", decodedMessage); - resolve(decodedMessage); - }) - .catch((error: any) => { - console.log("ERROR, proto couldn't be loaded", error); - reject(); - }); - }); -} - -type ProtoLoaderProps = { - onLoad: (protoMessage: any) => void; - messageType: MessageType; -}; - -function ProtoLoader(props: ProtoLoaderProps): React.ReactElement { - const [protoMessage, setProtoMessage] = useState(); - - async function processFile(event: React.FormEvent) { - try { - const target = event.target as HTMLInputElement & { files: FileList }; - if (!target.files || target.files.length <= 0) { - console.log("No file was uploaded"); - return; - } - - let contentBuffer = await readFileAsync(target.files[0]); - if (contentBuffer == null) { - throw new Error("Empty content"); - } - let protoMessage = await readProto( - new Uint8Array(contentBuffer), - "image_sequence_localizer." + props.messageType.toString() - ); - console.log("Read proto message", protoMessage); - setProtoMessage(protoMessage); - } catch (err) { - console.log("File was not processed", err); - } - } - - const { onLoad } = props; - useEffect(() => { - console.log("Proto state changed", protoMessage); - onLoad(protoMessage); - }, [protoMessage, onLoad]); - - return ( -
- - -
- ); -} - -export { ProtoLoader, MessageType }; diff --git a/src/viewer/src/context/ElementContext.tsx b/src/viewer/src/context/ElementContext.tsx new file mode 100644 index 0000000..f2d6361 --- /dev/null +++ b/src/viewer/src/context/ElementContext.tsx @@ -0,0 +1,42 @@ +import React, { createContext, useContext, ReactNode, useState } from "react"; +import { CostMatrixElement } from "../resources/costMatrix"; + +interface ElementContextProps { + globalSelectedElement?: CostMatrixElement; + setGlobalSelectedElement: React.Dispatch< + React.SetStateAction + >; +} + +const ElementContext = createContext( + undefined +); + +interface ElementProviderProps { + children: ReactNode; +} + +function ElementProvider(props: ElementProviderProps) { + const [globalSelectedElement, setGlobalSelectedElement] = + useState(); + + return ( + + {props.children} + + ); +} + +function useElementContext(): ElementContextProps { + const context = useContext(ElementContext); + + if (!context) { + throw new Error("useAppContext must be used within an AppProvider"); + } + + return context; +} + +export { ElementProvider, useElementContext }; From 11e03d771e4d7ff756a9a6df9527d3ccbde63798 Mon Sep 17 00:00:00 2001 From: olga Date: Wed, 6 Dec 2023 20:40:53 +0100 Subject: [PATCH 4/6] added focus lines. They are synchronized between interactive cost matrix and images. Potentially introduced a bug. Image with idx 0 is not loading --- src/viewer/src/App.tsx | 2 +- .../src/components/CostMatrixComponent.tsx | 2 +- src/viewer/src/components/ImageCarousel.tsx | 20 ++++-- src/viewer/src/components/ImageCostMatrix.tsx | 4 +- .../src/components/ImagePreviewComponent.tsx | 31 ++++++++- .../src/components/InteractiveCostMatrix.tsx | 63 ++++++++++++++++++- src/viewer/src/context/ElementContext.tsx | 15 +++-- 7 files changed, 119 insertions(+), 18 deletions(-) diff --git a/src/viewer/src/App.tsx b/src/viewer/src/App.tsx index d6e6c86..596a834 100644 --- a/src/viewer/src/App.tsx +++ b/src/viewer/src/App.tsx @@ -36,7 +36,7 @@ function App() { />
{costMatrixProtoFile && ( } + control={} label="Matching result" /> void; }; function ImageCarousel(props: ImageCarouselProps) { @@ -59,7 +60,7 @@ function ImageCarousel(props: ImageCarouselProps) { setImages(imageList); }) .catch((error) => { - console.log("Image was not loaded", error); + console.warn("Image was not loaded", error); }); } }); @@ -91,17 +92,25 @@ function ImageCarousel(props: ImageCarouselProps) { } function handleNextClick() { - if (images == null) { + if (images == null || currentImageId == null) { return; } - setCurrentImageId(Math.min(currentImageId + 1, images.length - 1)); + const nextImageId = Math.min(currentImageId + 1, images.length - 1); + setCurrentImageId(nextImageId); + if (props.setSelectedImageId != null) { + props.setSelectedImageId(nextImageId); + } } function handlePrevClick() { - if (images == null) { + if (images == null || currentImageId == null) { return; } - setCurrentImageId(Math.max(currentImageId - 1, 0)); + const prevImageId = Math.max(currentImageId - 1, 0); + setCurrentImageId(prevImageId); + if (props.setSelectedImageId != null) { + props.setSelectedImageId(prevImageId); + } } return ( @@ -119,6 +128,7 @@ function ImageCarousel(props: ImageCarouselProps) {
)} {images && + currentImageId && images.length > currentImageId && images[currentImageId] !== undefined && (
diff --git a/src/viewer/src/components/ImageCostMatrix.tsx b/src/viewer/src/components/ImageCostMatrix.tsx index 9845b0e..b2ef903 100644 --- a/src/viewer/src/components/ImageCostMatrix.tsx +++ b/src/viewer/src/components/ImageCostMatrix.tsx @@ -37,9 +37,9 @@ const ZoomTooltip = forwardRef((props: ZoomTooltipProps, pixelZoomRef: any) => { position: "absolute", top: props.coordY + "px", left: props.coordX + "px", - border: "3px solid pink", + boxShadow: "0px 0px 15px lavender", visibility: props.coordX && props.coordY ? "visible" : "hidden", - zIndex: 2 + zIndex: 2, }} > diff --git a/src/viewer/src/components/ImagePreviewComponent.tsx b/src/viewer/src/components/ImagePreviewComponent.tsx index ae9d597..399bc77 100644 --- a/src/viewer/src/components/ImagePreviewComponent.tsx +++ b/src/viewer/src/components/ImagePreviewComponent.tsx @@ -1,4 +1,5 @@ import React from "react"; + import { ImageCarousel } from "./ImageCarousel"; import { useElementContext } from "../context/ElementContext"; @@ -14,12 +15,36 @@ function ImagePreviewComponent( const { globalSelectedElement, setGlobalSelectedElement } = useElementContext(); + function updateGlobalQueryId(queryId?: number) { + if (queryId == null) { + return; + } + if (globalSelectedElement != null) { + setGlobalSelectedElement({ + queryId: queryId, + referenceId: globalSelectedElement?.referenceId, + }); + } + } + + function updateGlobalRefId(refId?: number) { + if (refId == null) { + return; + } + if (globalSelectedElement != null) { + setGlobalSelectedElement({ + queryId: globalSelectedElement.queryId, + referenceId: refId, + }); + } + } + return (
diff --git a/src/viewer/src/components/InteractiveCostMatrix.tsx b/src/viewer/src/components/InteractiveCostMatrix.tsx index 000d783..70ca22b 100644 --- a/src/viewer/src/components/InteractiveCostMatrix.tsx +++ b/src/viewer/src/components/InteractiveCostMatrix.tsx @@ -3,7 +3,7 @@ import { useState, useRef, useEffect, useCallback } from "react"; import { CostMatrix, CostMatrixElement } from "../resources/costMatrix"; import { ZoomBlockParams } from "./ImageCostMatrix"; -import { useElementContext } from "../context/ElementContext"; +import { useElementContext, SelectedElement } from "../context/ElementContext"; import * as d3 from "d3"; import { MatchingResultElement } from "../resources/readers"; @@ -112,6 +112,38 @@ function InteractiveCostMatrix( }); }, []); + const moveSliders = useCallback( + (element: SelectedElement) => { + const cellInZoomX = + (element.referenceId - props.zoomBlock.topLeftX) * cellSize + + cellSize / 2; + const cellInZoomY = + (element.queryId - props.zoomBlock.topLeftY) * cellSize + cellSize / 2; + + d3.select(svgRef.current) + .select(".ySlider") + .transition() + .duration(500) // Animation duration in milliseconds + .attr("y1", cellInZoomY) + .attr("y2", cellInZoomY); + d3.select(svgRef.current) + .select(".xSlider") + .transition() + .duration(500) // Animation duration in milliseconds + .attr("x1", cellInZoomX) + .attr("x2", cellInZoomX); + }, + [props.zoomBlock, cellSize] + ); + + // Move sliders when somebody changed the global selected element context + useEffect(() => { + if (globalSelectedElement == null) { + return; + } + moveSliders(globalSelectedElement); + }, [globalSelectedElement, moveSliders]); + const onCellUnHover = useCallback((event: any) => { setTooltipVisible(false); event.target.setAttribute("opacity", 1.0); @@ -119,7 +151,11 @@ function InteractiveCostMatrix( const onCellClick = useCallback( (event: any, cell: CostMatrixElement) => { - setGlobalSelectedElement(cell); + // moveSliders({ queryId: cell.queryId, referenceId: cell.refId }); + setGlobalSelectedElement({ + queryId: cell.queryId, + referenceId: cell.refId, + }); }, [setGlobalSelectedElement] ); @@ -158,6 +194,29 @@ function InteractiveCostMatrix( onCellClick, ]); + useEffect(() => { + const valuesGroup = d3.select(svgRef.current).select(".matrixEntries"); + valuesGroup + .append("line") + .attr("class", "ySlider") + .attr("id", "ySlider") + .attr("x1", 0) + .attr("y1", 0) + .attr("x2", height) + .attr("y2", 0) + .attr("stroke", "#1e81b0") + .attr("stroke-width", 2); + valuesGroup + .append("line") + .attr("class", "xSlider") + .attr("x1", 0) + .attr("y1", 0) + .attr("x2", 0) + .attr("y2", height) + .attr("stroke", "#eab676") + .attr("stroke-width", 2); + }, [props.costMatrix, props.zoomBlock]); + useEffect(() => { if (props.matches == null) { return; diff --git a/src/viewer/src/context/ElementContext.tsx b/src/viewer/src/context/ElementContext.tsx index f2d6361..7c967b4 100644 --- a/src/viewer/src/context/ElementContext.tsx +++ b/src/viewer/src/context/ElementContext.tsx @@ -1,10 +1,14 @@ import React, { createContext, useContext, ReactNode, useState } from "react"; -import { CostMatrixElement } from "../resources/costMatrix"; + +type SelectedElement = { + queryId: number; + referenceId: number; +}; interface ElementContextProps { - globalSelectedElement?: CostMatrixElement; + globalSelectedElement?: SelectedElement; setGlobalSelectedElement: React.Dispatch< - React.SetStateAction + React.SetStateAction >; } @@ -18,7 +22,7 @@ interface ElementProviderProps { function ElementProvider(props: ElementProviderProps) { const [globalSelectedElement, setGlobalSelectedElement] = - useState(); + useState(); return ( Date: Mon, 18 Dec 2023 11:33:40 +0100 Subject: [PATCH 5/6] found the bug with loading 0th image --- src/viewer/src/components/ImageCarousel.tsx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/viewer/src/components/ImageCarousel.tsx b/src/viewer/src/components/ImageCarousel.tsx index 84ecae4..66b9230 100644 --- a/src/viewer/src/components/ImageCarousel.tsx +++ b/src/viewer/src/components/ImageCarousel.tsx @@ -127,10 +127,10 @@ function ImageCarousel(props: ImageCarouselProps) {
)} - {images && - currentImageId && + {images && currentImageId !== undefined && images.length > currentImageId && - images[currentImageId] !== undefined && ( + images[currentImageId] !== undefined && + (
- )} + )}
); From d24af981f78058dc67b52396bfaf067105af4184 Mon Sep 17 00:00:00 2001 From: olga Date: Mon, 18 Dec 2023 14:27:16 +0100 Subject: [PATCH 6/6] final styling --- src/viewer/src/components/CostMatrixComponent.tsx | 1 + src/viewer/src/components/ImageCarousel.css | 6 +++--- src/viewer/src/components/InteractiveCostMatrix.tsx | 3 +-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/viewer/src/components/CostMatrixComponent.tsx b/src/viewer/src/components/CostMatrixComponent.tsx index 5e8352a..0801219 100644 --- a/src/viewer/src/components/CostMatrixComponent.tsx +++ b/src/viewer/src/components/CostMatrixComponent.tsx @@ -132,6 +132,7 @@ function CostMatrixComponent(props: CostMatrixProps): React.ReactElement { flexDirection: "row", flexWrap: "wrap", justifyContent: "center", + backgroundColor: "ghostwhite" }} >
diff --git a/src/viewer/src/components/ImageCarousel.css b/src/viewer/src/components/ImageCarousel.css index f756d3c..2c11bdb 100644 --- a/src/viewer/src/components/ImageCarousel.css +++ b/src/viewer/src/components/ImageCarousel.css @@ -12,7 +12,7 @@ .slide{ border-radius: 0.5rem; - box-shadow: 0px 0px 7px #666; + box-shadow: 0px 0px 7px #818da1; width: 100%; height:100%; } @@ -21,7 +21,7 @@ position: absolute; background-color:rgba(255,255,255,.8); top: -1rem; - box-shadow: 0px 0px 7px #666; + box-shadow: 0px 0px 7px #818da1; padding: 2px 10px 2px 10px; border-radius: 2px; } @@ -32,7 +32,7 @@ height: 3rem; color: black; background-color: white; - box-shadow: 0px 0px 7px #666; + box-shadow: 0px 0px 7px #818da1; border-radius: 0.5rem; } .arrow:hover{ diff --git a/src/viewer/src/components/InteractiveCostMatrix.tsx b/src/viewer/src/components/InteractiveCostMatrix.tsx index 70ca22b..aaacfa2 100644 --- a/src/viewer/src/components/InteractiveCostMatrix.tsx +++ b/src/viewer/src/components/InteractiveCostMatrix.tsx @@ -151,7 +151,6 @@ function InteractiveCostMatrix( const onCellClick = useCallback( (event: any, cell: CostMatrixElement) => { - // moveSliders({ queryId: cell.queryId, referenceId: cell.refId }); setGlobalSelectedElement({ queryId: cell.queryId, referenceId: cell.refId, @@ -284,7 +283,7 @@ function InteractiveCostMatrix(