diff --git a/.gitignore b/.gitignore index af39a2ec..4bed8c4c 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,7 @@ *.class # Log file -*.log +*.log* *.log.lck # BlueJ files diff --git a/aitoigt/detect.py b/aitoigt/detect.py new file mode 100644 index 00000000..83d31523 --- /dev/null +++ b/aitoigt/detect.py @@ -0,0 +1,353 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license +""" +Run YOLOv5 detection inference on images, videos, directories, globs, YouTube, webcam, streams, etc. + +Usage - sources: + $ python detect.py --weights yolov5s.pt --source 0 # webcam + img.jpg # image + vid.mp4 # video + screen # screenshot + path/ # directory + list.txt # list of images + list.streams # list of streams + 'path/*.jpg' # glob + 'https://youtu.be/LNwODJXcvt4' # YouTube + 'rtsp://example.com/media.mp4' # RTSP, RTMP, HTTP stream + +Usage - formats: + $ python detect.py --weights yolov5s.pt # PyTorch + yolov5s.torchscript # TorchScript + yolov5s.onnx # ONNX Runtime or OpenCV DNN with --dnn + yolov5s_openvino_model # OpenVINO + yolov5s.engine # TensorRT + yolov5s.mlmodel # CoreML (macOS-only) + yolov5s_saved_model # TensorFlow SavedModel + yolov5s.pb # TensorFlow GraphDef + yolov5s.tflite # TensorFlow Lite + yolov5s_edgetpu.tflite # TensorFlow Edge TPU + yolov5s_paddle_model # PaddlePaddle +""" + +import argparse +import csv +import numpy as np +import os +import platform +import sys +import pyigtl as igtl +from pathlib import Path + +import pyvirtualcam + +CAM = pyvirtualcam.Camera(width=640, height=480, fps=20) + +import torch + +FILE = Path(__file__).resolve() +ROOT = FILE.parents[0] # YOLOv5 root directory +if str(ROOT) not in sys.path: + sys.path.append(str(ROOT)) # add ROOT to PATH +ROOT = Path(os.path.relpath(ROOT, Path.cwd())) # relative + +last_matrix = np.eye(4) + +from ultralytics.utils.plotting import Annotator, colors, save_one_box + +from models.common import DetectMultiBackend +from utils.dataloaders import IMG_FORMATS, VID_FORMATS, LoadImages, LoadScreenshots, LoadStreams +from utils.general import ( + LOGGER, + Profile, + check_file, + check_img_size, + check_imshow, + check_requirements, + colorstr, + cv2, + increment_path, + non_max_suppression, + print_args, + scale_boxes, + strip_optimizer, + xyxy2xywh, +) +from utils.torch_utils import select_device, smart_inference_mode + + +@smart_inference_mode() +def run( + weights=ROOT / "yolov5s.pt", # model path or triton URL + source=ROOT / "data/images", # file/dir/URL/glob/screen/0(webcam) + data=ROOT / "data/coco128.yaml", # dataset.yaml path + imgsz=(640, 640), # inference size (height, width) + conf_thres=0.25, # confidence threshold + iou_thres=0.45, # NMS IOU threshold + max_det=1000, # maximum detections per image + device="", # cuda device, i.e. 0 or 0,1,2,3 or cpu + view_img=False, # show results + open_igt=False, # stream data thru igtlink + save_txt=True, # save results to *.txt + save_csv=False, # save results in CSV format + save_conf=False, # save confidences in --save-txt labels + save_crop=False, # save cropped prediction boxes + nosave=False, # do not save images/videos + classes=None, # filter by class: --class 0, or --class 0 2 3 + agnostic_nms=False, # class-agnostic NMS + augment=False, # augmented inference + visualize=False, # visualize features + update=False, # update all models + project=ROOT / "runs/detect", # save results to project/name + name="exp", # save results to project/name + exist_ok=False, # existing project/name ok, do not increment + line_thickness=3, # bounding box thickness (pixels) + hide_labels=False, # hide labels + hide_conf=False, # hide confidences + half=False, # use FP16 half-precision inference + dnn=False, # use OpenCV DNN for ONNX inference + vid_stride=1, # video frame-rate stride +): + + if (open_igt): + # Create an OpenIGTLink object + server = igtl.OpenIGTLinkServer(port=18944) + + # Check if the server is running + if not server.is_connected(): + print("Server started. Waiting for connections...") + + source = str(source) + save_img = not nosave and not source.endswith(".txt") # save inference images + is_file = Path(source).suffix[1:] in (IMG_FORMATS + VID_FORMATS) + is_url = source.lower().startswith(("rtsp://", "rtmp://", "http://", "https://")) + webcam = source.isnumeric() or source.endswith(".streams") or (is_url and not is_file) + screenshot = source.lower().startswith("screen") + if is_url and is_file: + source = check_file(source) # download + + # Directories + save_dir = increment_path(Path(project) / name, exist_ok=exist_ok) # increment run + (save_dir / "labels" if save_txt else save_dir).mkdir(parents=True, exist_ok=True) # make dir + + # Load model + device = select_device(device) + model = DetectMultiBackend(weights, device=device, dnn=dnn, data=data, fp16=half) + stride, names, pt = model.stride, model.names, model.pt + imgsz = check_img_size(imgsz, s=stride) # check image size + + # Dataloader + bs = 1 # batch_size + if webcam: + view_img = check_imshow(warn=True) + dataset = LoadStreams(source, img_size=imgsz, stride=stride, auto=pt, vid_stride=vid_stride) + bs = len(dataset) + elif screenshot: + dataset = LoadScreenshots(source, img_size=imgsz, stride=stride, auto=pt) + else: + dataset = LoadImages(source, img_size=imgsz, stride=stride, auto=pt, vid_stride=vid_stride) + vid_path, vid_writer = [None] * bs, [None] * bs + + # Run inference + model.warmup(imgsz=(1 if pt or model.triton else bs, 3, *imgsz)) # warmup + seen, windows, dt = 0, [], (Profile(device=device), Profile(device=device), Profile(device=device)) + for path, im, im0s, vid_cap, s in dataset: + with dt[0]: + im = torch.from_numpy(im).to(model.device) + im = im.half() if model.fp16 else im.float() # uint8 to fp16/32 + im /= 255 # 0 - 255 to 0.0 - 1.0 + if len(im.shape) == 3: + im = im[None] # expand for batch dim + if model.xml and im.shape[0] > 1: + ims = torch.chunk(im, im.shape[0], 0) + + # Inference + with dt[1]: + visualize = increment_path(save_dir / Path(path).stem, mkdir=True) if visualize else False + if model.xml and im.shape[0] > 1: + pred = None + for image in ims: + if pred is None: + pred = model(image, augment=augment, visualize=visualize).unsqueeze(0) + else: + pred = torch.cat((pred, model(image, augment=augment, visualize=visualize).unsqueeze(0)), dim=0) + pred = [pred, None] + else: + pred = model(im, augment=augment, visualize=visualize) + # NMS + with dt[2]: + pred = non_max_suppression(pred, conf_thres, iou_thres, classes, agnostic_nms, max_det=max_det) + + # Second-stage classifier (optional) + # pred = utils.general.apply_classifier(pred, classifier_model, im, im0s) + + # Define the path for the CSV file + csv_path = save_dir / "predictions.csv" + + # Create or append to the CSV file + def write_to_csv(image_name, prediction, confidence): + """Writes prediction data for an image to a CSV file, appending if the file exists.""" + data = {"Image Name": image_name, "Prediction": prediction, "Confidence": confidence} + with open(csv_path, mode="a", newline="") as f: + writer = csv.DictWriter(f, fieldnames=data.keys()) + if not csv_path.is_file(): + writer.writeheader() + writer.writerow(data) + + # Process predictions + for i, det in enumerate(pred): # per image + seen += 1 + if webcam: # batch_size >= 1 + p, im0, frame = path[i], im0s[i].copy(), dataset.count + s += f"{i}: " + else: + p, im0, frame = path, im0s.copy(), getattr(dataset, "frame", 0) + + p = Path(p) # to Path + save_path = str(save_dir / p.name) # im.jpg + txt_path = str(save_dir / "labels" / p.stem) + ("" if dataset.mode == "image" else f"_{frame}") # im.txt + s += "%gx%g " % im.shape[2:] # print string + gn = torch.tensor(im0.shape)[[1, 0, 1, 0]] # normalization gain whwh + imc = im0.copy() if save_crop else im0 # for save_crop + annotator = Annotator(im0, line_width=line_thickness, example=str(names)) + if len(det): + # Rescale boxes from img_size to im0 size + det[:, :4] = scale_boxes(im.shape[2:], det[:, :4], im0.shape).round() + + # Print results + for c in det[:, 5].unique(): + n = (det[:, 5] == c).sum() # detections per class + s += f"{n} {names[int(c)]}{'s' * (n > 1)}, " # add to string + + # Write results + for *xyxy, conf, cls in reversed(det): + c = int(cls) # integer class + label = names[c] if hide_conf else f"{names[c]}" + confidence = float(conf) + confidence_str = f"{confidence:.2f}" + + if save_csv: + write_to_csv(p.name, label, confidence_str) + + if save_txt: # Write to file + xywh = (xyxy2xywh(torch.tensor(xyxy).view(1, 4)) / gn).view(-1).tolist() # normalized xywh + line = (cls, *xywh, conf) if save_conf else (cls, *xywh) # label format + with open(f"{txt_path}.txt", "a") as f: + f.write(("%g " * len(line)).rstrip() % line + "\n") + + if save_img or save_crop or view_img: # Add bbox to image + c = int(cls) # integer class + label = None if hide_labels else (names[c] if hide_conf else f"{names[c]} {conf:.2f}") + annotator.box_label(xyxy, label, color=colors(c, True)) + if save_crop: + save_one_box(xyxy, imc, file=save_dir / "crops" / names[c] / f"{p.stem}.jpg", BGR=True) + + if open_igt: + + # Generate matrix + global last_matrix + matrix = np.eye(4) + + # Set position + matrix[0, 3] = xyxy[0] + matrix[1, 3] = xyxy[1] * -1 + 500 + + # Set orientation + direction = last_matrix[:3, 3] - matrix[:3, 3] + angle = (-np.pi) + np.arctan2(direction[1], direction[0]) # -90 degree offset because -X is the forward direction of the IGTP pointer model + rotation_matrix = np.array([[np.cos(angle), -np.sin(angle), 0], [np.sin(angle), np.cos(angle), 0], [0, 0, 1]]) + matrix[:3, :3] = rotation_matrix + + # Send transform message + last_matrix = matrix + transform_message = igtl.TransformMessage(matrix, device_name="ImageToReference", timestamp=1) + server.send_message(transform_message) + + # Stream results + im0 = annotator.result() + if view_img: + if platform.system() == "Linux" and p not in windows: + windows.append(p) + cv2.namedWindow(str(p), cv2.WINDOW_NORMAL | cv2.WINDOW_KEEPRATIO) # allow window resize (Linux) + cv2.resizeWindow(str(p), im0.shape[1], im0.shape[0]) + cv2.imshow(str(p), im0) + cv2.waitKey(1) # 1 millisecond + # Send RGB image to virtual camera + im0VIRT = cv2.cvtColor(im0, cv2.COLOR_BGR2RGB) + CAM.send(im0VIRT) + # Save results (image with detections) + if save_img: + if dataset.mode == "image": + cv2.imwrite(save_path, im0) + else: # 'video' or 'stream' + if vid_path[i] != save_path: # new video + vid_path[i] = save_path + if isinstance(vid_writer[i], cv2.VideoWriter): + vid_writer[i].release() # release previous video writer + if vid_cap: # video + fps = vid_cap.get(cv2.CAP_PROP_FPS) + w = int(vid_cap.get(cv2.CAP_PROP_FRAME_WIDTH)) + h = int(vid_cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + else: # stream + fps, w, h = 30, im0.shape[1], im0.shape[0] + save_path = str(Path(save_path).with_suffix(".mp4")) # force *.mp4 suffix on results videos + vid_writer[i] = cv2.VideoWriter(save_path, cv2.VideoWriter_fourcc(*"mp4v"), fps, (w, h)) + vid_writer[i].write(im0) + + # Print time (inference-only) + LOGGER.info(f"{s}{'' if len(det) else '(no detections), '}{dt[1].dt * 1E3:.1f}ms") + + # Print results + t = tuple(x.t / seen * 1e3 for x in dt) # speeds per image + LOGGER.info(f"Speed: %.1fms pre-process, %.1fms inference, %.1fms NMS per image at shape {(1, 3, *imgsz)}" % t) + if save_txt or save_img: + s = f"\n{len(list(save_dir.glob('labels/*.txt')))} labels saved to {save_dir / 'labels'}" if save_txt else "" + LOGGER.info(f"Results saved to {colorstr('bold', save_dir)}{s}") + if update: + strip_optimizer(weights[0]) # update model (to fix SourceChangeWarning) + + +def parse_opt(): + """Parses command-line arguments for YOLOv5 detection, setting inference options and model configurations.""" + parser = argparse.ArgumentParser() + parser.add_argument("--weights", nargs="+", type=str, default=ROOT / "yolov5s.pt", help="model path or triton URL") + parser.add_argument("--source", type=str, default=ROOT / "data/images", help="file/dir/URL/glob/screen/0(webcam)") + parser.add_argument("--data", type=str, default=ROOT / "data/coco128.yaml", help="(optional) dataset.yaml path") + parser.add_argument("--imgsz", "--img", "--img-size", nargs="+", type=int, default=[640], help="inference size h,w") + parser.add_argument("--conf-thres", type=float, default=0.25, help="confidence threshold") + parser.add_argument("--iou-thres", type=float, default=0.45, help="NMS IoU threshold") + parser.add_argument("--max-det", type=int, default=1000, help="maximum detections per image") + parser.add_argument("--device", default="", help="cuda device, i.e. 0 or 0,1,2,3 or cpu") + parser.add_argument("--view-img", action="store_true", help="show results") + parser.add_argument("--open-igt", action="store_true", help="stream data thru igtlink") + parser.add_argument("--save-txt", action="store_true", help="save results to *.txt") + parser.add_argument("--save-csv", action="store_true", help="save results in CSV format") + parser.add_argument("--save-conf", action="store_true", help="save confidences in --save-txt labels") + parser.add_argument("--save-crop", action="store_true", help="save cropped prediction boxes") + parser.add_argument("--nosave", action="store_true", help="do not save images/videos") + parser.add_argument("--classes", nargs="+", type=int, help="filter by class: --classes 0, or --classes 0 2 3") + parser.add_argument("--agnostic-nms", action="store_true", help="class-agnostic NMS") + parser.add_argument("--augment", action="store_true", help="augmented inference") + parser.add_argument("--visualize", action="store_true", help="visualize features") + parser.add_argument("--update", action="store_true", help="update all models") + parser.add_argument("--project", default=ROOT / "runs/detect", help="save results to project/name") + parser.add_argument("--name", default="exp", help="save results to project/name") + parser.add_argument("--exist-ok", action="store_true", help="existing project/name ok, do not increment") + parser.add_argument("--line-thickness", default=3, type=int, help="bounding box thickness (pixels)") + parser.add_argument("--hide-labels", default=False, action="store_true", help="hide labels") + parser.add_argument("--hide-conf", default=False, action="store_true", help="hide confidences") + parser.add_argument("--half", action="store_true", help="use FP16 half-precision inference") + parser.add_argument("--dnn", action="store_true", help="use OpenCV DNN for ONNX inference") + parser.add_argument("--vid-stride", type=int, default=1, help="video frame-rate stride") + opt = parser.parse_args() + opt.imgsz *= 2 if len(opt.imgsz) == 1 else 1 # expand + print_args(vars(opt)) + return opt + +def main(opt): + """Executes YOLOv5 model inference with given options, checking requirements before running the model.""" + check_requirements(ROOT / "requirements.txt", exclude=("tensorboard", "thop")) + run(**vars(opt)) + + +if __name__ == "__main__": + opt = parse_opt() + main(opt) \ No newline at end of file diff --git a/aitoigt/test.py b/aitoigt/test.py new file mode 100644 index 00000000..1fe829e4 --- /dev/null +++ b/aitoigt/test.py @@ -0,0 +1,65 @@ +""" +============================ +Tracked image data server +============================ + +Simple application that starts a server that sends images, transforms, and strings + +""" + +import pyigtl # pylint: disable=import-error +from math import cos, sin, pi +from time import sleep +import numpy as np + +server = pyigtl.OpenIGTLinkServer(port=18944, local_server=True) + +image_size = [400, 200] + +timestep = 0 +last_matrix = np.eye(4) + +while True: + + if not server.is_connected(): + # Wait for client to connect + sleep(0.1) + continue + + # Init vars + timestep += 1 + theta = timestep * 0.01 + + # Generate transform + matrix = np.eye(4) + + # Set position + matrix[0, 3] = sin(theta) * 100.0 + matrix[1, 3] = sin(theta) * cos(theta) * 100.0 + + # Set orientation + direction = last_matrix[:3, 3] - matrix[:3, 3] + angle = (-np.pi) + np.arctan2(direction[1], direction[0]) # -90 degree offset because -X is the forward direction of the IGTP pointer model + nlah = np.array([[np.cos(angle), -np.sin(angle), 0], [np.sin(angle), np.cos(angle), 0], [0, 0, 1]]) + matrix[:3, :3] = nlah + + # Debugging + print(f"Y Coord: {direction[1]}") + print(f"X Coord: {direction[0]}") + print(f"Angle: {angle}") + + # Send transform message + last_matrix = matrix + transform_message = pyigtl.TransformMessage(matrix, device_name="ImageToReference", timestamp=1) + + # Send messages + server.send_message(transform_message) + + # Print received messages + messages = server.get_latest_messages() + for message in messages: + print(message.device_name) + + # Do not flood the message queue, + # but allow a little time for background network transfer + sleep(0.01) \ No newline at end of file diff --git a/logging.log.1 b/logging.log.1 new file mode 100644 index 00000000..fae1c313 --- /dev/null +++ b/logging.log.1 @@ -0,0 +1,176 @@ +Apr 11, 2024 2:04:20 AM javafx.scene.Node$FocusedProperty markInvalid +FINE: ReadOnlyBooleanProperty [bean: TabPane[id=tabPane, styleClass=tab-pane], name: focused, value: true] focused=true +Apr 11, 2024 2:04:20 AM javafx.scene.Scene$12 invalidated +FINE: Changed focus from null to TabPane[id=tabPane, styleClass=tab-pane] +Apr 11, 2024 2:04:21 AM javafx.scene.Node$FocusedProperty markInvalid +FINE: ReadOnlyBooleanProperty [bean: TabPane[id=tabPane, styleClass=tab-pane], name: focused, value: false] focused=false +Apr 11, 2024 2:04:21 AM javafx.scene.Node$FocusedProperty markInvalid +FINE: ReadOnlyBooleanProperty [bean: ToggleButton@61665b10[styleClass=toggle-button green-toggle-button]'Connect via OpenIGTLink (localhost)', name: focused, value: true] focused=true +Apr 11, 2024 2:04:21 AM javafx.scene.Scene$12 invalidated +FINE: Changed focus from TabPane[id=tabPane, styleClass=tab-pane] to ToggleButton@61665b10[styleClass=toggle-button green-toggle-button]'Connect via OpenIGTLink (localhost)' +Apr 11, 2024 2:04:21 AM inputOutput.OpenIGTLinkConnection +INFO: Starting OIGTL client +Apr 11, 2024 2:04:21 AM org.medcare.igtl.network.OpenIGTClient +SEVERE: OpenIGTClient Exception while creating socket +java.net.ConnectException: Connection refused: connect + at java.base/sun.nio.ch.Net.connect0(Native Method) + at java.base/sun.nio.ch.Net.connect(Net.java:579) + at java.base/sun.nio.ch.Net.connect(Net.java:568) + at java.base/sun.nio.ch.NioSocketImpl.connect(NioSocketImpl.java:588) + at java.base/java.net.SocksSocketImpl.connect(SocksSocketImpl.java:327) + at java.base/java.net.Socket.connect(Socket.java:633) + at java.base/java.net.Socket.connect(Socket.java:583) + at java.base/java.net.Socket.(Socket.java:507) + at java.base/java.net.Socket.(Socket.java:287) + at java.base/javax.net.DefaultSocketFactory.createSocket(SocketFactory.java:271) + at org.medcare.igtl.network.OpenIGTClient.(OpenIGTClient.java:80) + at org.medcare.igtl.network.GenericIGTLinkClient.(GenericIGTLinkClient.java:23) + at inputOutput.OpenIGTLinkConnection.(OpenIGTLinkConnection.java:46) + at inputOutput.OIGTTrackingDataSource.connect(OIGTTrackingDataSource.java:31) + at controller.TrackingDataController.lambda$onConnectButtonClicked$1(TrackingDataController.java:157) + at java.base/java.lang.Thread.run(Thread.java:833) + +Apr 11, 2024 2:04:21 AM org.medcare.igtl.network.GenericIGTLinkClient +FINE: Starting GenericIGTLinkClient +Apr 11, 2024 2:04:21 AM mainMethod.App lambda$start$0 +SEVERE: Uncaught exception in thread Thread-4 +java.lang.NullPointerException: Cannot invoke "java.net.Socket.isClosed()" because "this.socket" is null + at org.medcare.igtl.network.OpenIGTClient.isConnected(OpenIGTClient.java:246) + at org.medcare.igtl.network.GenericIGTLinkClient$Sender.run(GenericIGTLinkClient.java:246) + +Apr 11, 2024 2:04:22 AM javafx.scene.Node$FocusedProperty markInvalid +FINE: ReadOnlyBooleanProperty [bean: ToggleButton@61665b10[styleClass=toggle-button green-toggle-button]'Connect via OpenIGTLink (localhost)', name: focused, value: false] focused=false +Apr 11, 2024 2:04:22 AM javafx.scene.Node$FocusedProperty markInvalid +FINE: ReadOnlyBooleanProperty [bean: Button[id=visualizeTrackingBtn, styleClass=button]'Start Tracking and Visualization', name: focused, value: true] focused=true +Apr 11, 2024 2:04:22 AM javafx.scene.Scene$12 invalidated +FINE: Changed focus from ToggleButton@61665b10[styleClass=toggle-button green-toggle-button]'Connect via OpenIGTLink (localhost)' to Button[id=visualizeTrackingBtn, styleClass=button]'Start Tracking and Visualization' +Apr 11, 2024 2:04:22 AM algorithm.TrackingDataManager getNextData +WARNING: Toollist is empty. +Apr 11, 2024 2:04:22 AM algorithm.TrackingDataManager getNextData +WARNING: Toollist is empty. +Apr 11, 2024 2:04:22 AM algorithm.VisualizationManager loadLastSTLModels +INFO: STL file read from: C:\GitHub\IGTPrototypingTool\src\test\resources\Tool13_v2.stl +Apr 11, 2024 2:04:22 AM algorithm.VisualizationManager loadLastSTLModels +INFO: STL file read from: C:\GitHub\IGTPrototypingTool\src\test\resources\Phantom_final-decimated25000.stl +Apr 11, 2024 2:04:22 AM algorithm.TrackingDataManager getNextData +WARNING: Toollist is empty. +Apr 11, 2024 2:04:22 AM algorithm.TrackingDataManager getNextData +WARNING: Toollist is empty. +Apr 11, 2024 2:04:22 AM javafx.scene.Node$FocusedProperty markInvalid +FINE: ReadOnlyBooleanProperty [bean: Button[id=visualizeTrackingBtn, styleClass=button]'Start Tracking and Visualization', name: focused, value: false] focused=false +Apr 11, 2024 2:04:22 AM javafx.scene.Scene$12 invalidated +FINE: Changed focus from Button[id=visualizeTrackingBtn, styleClass=button]'Start Tracking and Visualization' to null +Apr 11, 2024 2:04:22 AM javafx.scene.Node$FocusedProperty markInvalid +FINE: ReadOnlyBooleanProperty [bean: ToggleButton[id=freezeTglBtn, styleClass=toggle-button]'Freeze / Unfreeze', name: focused, value: true] focused=true +Apr 11, 2024 2:04:22 AM javafx.scene.Scene$12 invalidated +FINE: Changed focus from null to ToggleButton[id=freezeTglBtn, styleClass=toggle-button]'Freeze / Unfreeze' +Apr 11, 2024 2:04:22 AM algorithm.TrackingDataManager getNextData +WARNING: Toollist is empty. +Apr 11, 2024 2:04:23 AM algorithm.TrackingDataManager getNextData +WARNING: Toollist is empty. +Apr 11, 2024 2:04:23 AM algorithm.TrackingDataManager getNextData +WARNING: Toollist is empty. +Apr 11, 2024 2:04:23 AM algorithm.TrackingDataManager getNextData +WARNING: Toollist is empty. +Apr 11, 2024 2:04:23 AM algorithm.TrackingDataManager getNextData +WARNING: Toollist is empty. +Apr 11, 2024 2:04:23 AM algorithm.TrackingDataManager getNextData +WARNING: Toollist is empty. +Apr 11, 2024 2:04:23 AM algorithm.TrackingDataManager getNextData +WARNING: Toollist is empty. +Apr 11, 2024 2:04:23 AM algorithm.TrackingDataManager getNextData +WARNING: Toollist is empty. +Apr 11, 2024 2:04:23 AM javafx.scene.Node$FocusedProperty markInvalid +FINE: ReadOnlyBooleanProperty [bean: ToggleButton[id=freezeTglBtn, styleClass=toggle-button]'Freeze / Unfreeze', name: focused, value: false] focused=false +Apr 11, 2024 2:04:23 AM javafx.scene.Node$FocusedProperty markInvalid +FINE: ReadOnlyBooleanProperty [bean: ToggleButton@61665b10[styleClass=toggle-button green-toggle-button]'Connect via OpenIGTLink (localhost)', name: focused, value: true] focused=true +Apr 11, 2024 2:04:23 AM javafx.scene.Scene$12 invalidated +FINE: Changed focus from ToggleButton[id=freezeTglBtn, styleClass=toggle-button]'Freeze / Unfreeze' to ToggleButton@61665b10[styleClass=toggle-button green-toggle-button]'Connect via OpenIGTLink (localhost)' +Apr 11, 2024 2:04:23 AM algorithm.TrackingDataManager getNextData +WARNING: Toollist is empty. +Apr 11, 2024 2:04:23 AM mainMethod.App lambda$start$0 +SEVERE: Uncaught exception in thread JavaFX Application Thread +java.lang.RuntimeException: java.lang.reflect.InvocationTargetException + at javafx.fxml/javafx.fxml.FXMLLoader$MethodHandler.invoke(FXMLLoader.java:1857) + at javafx.fxml/javafx.fxml.FXMLLoader$ControllerMethodEventHandler.handle(FXMLLoader.java:1724) + at javafx.base/com.sun.javafx.event.CompositeEventHandler.dispatchBubblingEvent(CompositeEventHandler.java:86) + at javafx.base/com.sun.javafx.event.EventHandlerManager.dispatchBubblingEvent(EventHandlerManager.java:234) + at javafx.base/com.sun.javafx.event.EventHandlerManager.dispatchBubblingEvent(EventHandlerManager.java:191) + at javafx.base/com.sun.javafx.event.CompositeEventDispatcher.dispatchBubblingEvent(CompositeEventDispatcher.java:59) + at javafx.base/com.sun.javafx.event.BasicEventDispatcher.dispatchEvent(BasicEventDispatcher.java:58) + at javafx.base/com.sun.javafx.event.EventDispatchChainImpl.dispatchEvent(EventDispatchChainImpl.java:114) + at javafx.base/com.sun.javafx.event.BasicEventDispatcher.dispatchEvent(BasicEventDispatcher.java:56) + at javafx.base/com.sun.javafx.event.EventDispatchChainImpl.dispatchEvent(EventDispatchChainImpl.java:114) + at javafx.base/com.sun.javafx.event.BasicEventDispatcher.dispatchEvent(BasicEventDispatcher.java:56) + at javafx.base/com.sun.javafx.event.EventDispatchChainImpl.dispatchEvent(EventDispatchChainImpl.java:114) + at javafx.base/com.sun.javafx.event.BasicEventDispatcher.dispatchEvent(BasicEventDispatcher.java:56) + at javafx.base/com.sun.javafx.event.EventDispatchChainImpl.dispatchEvent(EventDispatchChainImpl.java:114) + at javafx.base/com.sun.javafx.event.BasicEventDispatcher.dispatchEvent(BasicEventDispatcher.java:56) + at javafx.base/com.sun.javafx.event.EventDispatchChainImpl.dispatchEvent(EventDispatchChainImpl.java:114) + at javafx.base/com.sun.javafx.event.BasicEventDispatcher.dispatchEvent(BasicEventDispatcher.java:56) + at javafx.base/com.sun.javafx.event.EventDispatchChainImpl.dispatchEvent(EventDispatchChainImpl.java:114) + at javafx.base/com.sun.javafx.event.EventUtil.fireEventImpl(EventUtil.java:74) + at javafx.base/com.sun.javafx.event.EventUtil.fireEvent(EventUtil.java:49) + at javafx.base/javafx.event.Event.fireEvent(Event.java:198) + at javafx.graphics/javafx.scene.Node.fireEvent(Node.java:8792) + at javafx.controls/javafx.scene.control.ToggleButton.fire(ToggleButton.java:257) + at javafx.controls/com.sun.javafx.scene.control.behavior.ButtonBehavior.mouseReleased(ButtonBehavior.java:208) + at javafx.controls/com.sun.javafx.scene.control.inputmap.InputMap.handle(InputMap.java:274) + at javafx.base/com.sun.javafx.event.CompositeEventHandler$NormalEventHandlerRecord.handleBubblingEvent(CompositeEventHandler.java:247) + at javafx.base/com.sun.javafx.event.CompositeEventHandler.dispatchBubblingEvent(CompositeEventHandler.java:80) + at javafx.base/com.sun.javafx.event.EventHandlerManager.dispatchBubblingEvent(EventHandlerManager.java:234) + at javafx.base/com.sun.javafx.event.EventHandlerManager.dispatchBubblingEvent(EventHandlerManager.java:191) + at javafx.base/com.sun.javafx.event.CompositeEventDispatcher.dispatchBubblingEvent(CompositeEventDispatcher.java:59) + at javafx.base/com.sun.javafx.event.BasicEventDispatcher.dispatchEvent(BasicEventDispatcher.java:58) + at javafx.base/com.sun.javafx.event.EventDispatchChainImpl.dispatchEvent(EventDispatchChainImpl.java:114) + at javafx.base/com.sun.javafx.event.BasicEventDispatcher.dispatchEvent(BasicEventDispatcher.java:56) + at javafx.base/com.sun.javafx.event.EventDispatchChainImpl.dispatchEvent(EventDispatchChainImpl.java:114) + at javafx.base/com.sun.javafx.event.BasicEventDispatcher.dispatchEvent(BasicEventDispatcher.java:56) + at javafx.base/com.sun.javafx.event.EventDispatchChainImpl.dispatchEvent(EventDispatchChainImpl.java:114) + at javafx.base/com.sun.javafx.event.BasicEventDispatcher.dispatchEvent(BasicEventDispatcher.java:56) + at javafx.base/com.sun.javafx.event.EventDispatchChainImpl.dispatchEvent(EventDispatchChainImpl.java:114) + at javafx.base/com.sun.javafx.event.BasicEventDispatcher.dispatchEvent(BasicEventDispatcher.java:56) + at javafx.base/com.sun.javafx.event.EventDispatchChainImpl.dispatchEvent(EventDispatchChainImpl.java:114) + at javafx.base/com.sun.javafx.event.BasicEventDispatcher.dispatchEvent(BasicEventDispatcher.java:56) + at javafx.base/com.sun.javafx.event.EventDispatchChainImpl.dispatchEvent(EventDispatchChainImpl.java:114) + at javafx.base/com.sun.javafx.event.EventUtil.fireEventImpl(EventUtil.java:74) + at javafx.base/com.sun.javafx.event.EventUtil.fireEvent(EventUtil.java:54) + at javafx.base/javafx.event.Event.fireEvent(Event.java:198) + at javafx.graphics/javafx.scene.Scene$MouseHandler.process(Scene.java:3897) + at javafx.graphics/javafx.scene.Scene.processMouseEvent(Scene.java:1878) + at javafx.graphics/javafx.scene.Scene$ScenePeerListener.mouseEvent(Scene.java:2623) + at javafx.graphics/com.sun.javafx.tk.quantum.GlassViewEventHandler$MouseEventNotification.run(GlassViewEventHandler.java:411) + at javafx.graphics/com.sun.javafx.tk.quantum.GlassViewEventHandler$MouseEventNotification.run(GlassViewEventHandler.java:301) + at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) + at javafx.graphics/com.sun.javafx.tk.quantum.GlassViewEventHandler.lambda$handleMouseEvent$2(GlassViewEventHandler.java:450) + at javafx.graphics/com.sun.javafx.tk.quantum.QuantumToolkit.runWithoutRenderLock(QuantumToolkit.java:424) + at javafx.graphics/com.sun.javafx.tk.quantum.GlassViewEventHandler.handleMouseEvent(GlassViewEventHandler.java:449) + at javafx.graphics/com.sun.glass.ui.View.handleMouseEvent(View.java:557) + at javafx.graphics/com.sun.glass.ui.View.notifyMouse(View.java:943) + at javafx.graphics/com.sun.glass.ui.win.WinApplication._runLoop(Native Method) + at javafx.graphics/com.sun.glass.ui.win.WinApplication.lambda$runLoop$3(WinApplication.java:184) + at java.base/java.lang.Thread.run(Thread.java:833) +Caused by: java.lang.reflect.InvocationTargetException + at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) + at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.base/java.lang.reflect.Method.invoke(Method.java:568) + at com.sun.javafx.reflect.Trampoline.invoke(MethodUtil.java:77) + at jdk.internal.reflect.GeneratedMethodAccessor2.invoke(Unknown Source) + at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.base/java.lang.reflect.Method.invoke(Method.java:568) + at javafx.base/com.sun.javafx.reflect.MethodUtil.invoke(MethodUtil.java:275) + at javafx.fxml/com.sun.javafx.fxml.MethodHelper.invoke(MethodHelper.java:84) + at javafx.fxml/javafx.fxml.FXMLLoader$MethodHandler.invoke(FXMLLoader.java:1854) + ... 58 more +Caused by: java.lang.NullPointerException: Cannot invoke "org.medcare.igtl.network.ResponseQueueManager.destroy()" because "this.responseQueue" is null + at org.medcare.igtl.network.OpenIGTClient.interrupt(OpenIGTClient.java:207) + at org.medcare.igtl.network.GenericIGTLinkClient.stopClient(GenericIGTLinkClient.java:160) + at inputOutput.OpenIGTLinkConnection.stop(OpenIGTLinkConnection.java:145) + at inputOutput.OIGTTrackingDataSource.closeConnection(OIGTTrackingDataSource.java:91) + at controller.TrackingDataController.disconnectSource(TrackingDataController.java:185) + at controller.TrackingDataController.onConnectButtonClicked(TrackingDataController.java:152) + ... 69 more + +Apr 11, 2024 2:04:24 AM javafx.scene.Node$FocusedProperty markInvalid +FINE: ReadOnlyBooleanProperty [bean: ToggleButton@61665b10[styleClass=toggle-button green-toggle-button]'Connect via OpenIGTLink (localhost)', name: focused, value: false] focused=false diff --git a/logging.log.1.lck b/logging.log.1.lck new file mode 100644 index 00000000..e69de29b diff --git a/src/main/java/controller/AiController.java b/src/main/java/controller/AiController.java new file mode 100644 index 00000000..6240f8e0 --- /dev/null +++ b/src/main/java/controller/AiController.java @@ -0,0 +1,489 @@ +package controller; + +import algorithm.*; +import inputOutput.ExportMeasurement; +import inputOutput.TransformationMatrix; +import inputOutput.VideoSource; +import javafx.animation.Animation; +import javafx.animation.KeyFrame; +import javafx.animation.Timeline; +import javafx.application.Platform; +import javafx.collections.FXCollections; +import javafx.collections.ObservableList; +import javafx.fxml.FXML; +import javafx.scene.chart.LineChart; +import javafx.scene.chart.NumberAxis; +import javafx.scene.chart.XYChart; +import javafx.scene.control.*; +import javafx.util.Duration; +import org.opencv.core.Core; +import org.opencv.core.CvType; +import org.opencv.core.Mat; +import org.opencv.core.Point3; +import userinterface.PlottableImage; + +import java.awt.image.BufferedImage; +import java.net.URL; +import java.util.*; +import java.util.prefs.Preferences; + +public class AiController implements Controller { + + private final ImageDataManager imageDataManager = new ImageDataManager(); + private final TrackingService trackingService = TrackingService.getInstance(); + private final Map deviceIdMapping = new LinkedHashMap<>(); + private static final Preferences userPreferencesGlobal = Preferences.userRoot().node("IGT_Settings"); + + @FXML + public ChoiceBox sourceChoiceBox; + @FXML + public CheckBox trackingConnectedStatusBox; + public ProgressIndicator connectionProgressSpinner; + @FXML + public PlottableImage videoImagePlot; + @FXML + public LineChart lineChart; + @FXML + public ChoiceBox ModeSelection; + @FXML + public TitledPane PathControlPanel; + @FXML + public Button clearAll; + + private Label statusLabel; + private Timeline videoTimeline; + private BufferedImage currentShowingImage; + private List lastTrackingData = new ArrayList<>(); + + // Used to crop the image to the actual content. Dirty describes whether the roi cache needs to be updated on the next transform, it's set when a new matrix is loaded + private int[] matrixRoi = new int[4]; + private boolean roiDirty = true; + + private TransformationMatrix transformationMatrix = new TransformationMatrix(); + + private final ObservableList> dataSeries = FXCollections.observableArrayList(); + + private final ObservableList clicked_image_points = FXCollections.observableArrayList(); + private final ObservableList clicked_tracker_points = FXCollections.observableArrayList(); + private Mat cachedTransformMatrix = null; + + private final XYChart.Series referencePoint = new XYChart.Series(); + private XYChart.Series trackingPoint = new XYChart.Series(); + private final ArrayList> referencePointsListPath = new ArrayList>(); + private final ObservableList> lineDataSeries = FXCollections.observableArrayList(); + + NumberAxis xAxis = new NumberAxis(-500, 500, 100); + NumberAxis yAxis = new NumberAxis(-500, 500, 100); + + + @Override + public void initialize(URL location, ResourceBundle resources) { + registerController(); + + trackingService.registerObserver((sourceChanged,dataServiceChanged,timelineChanged) -> updateTrackingInformation()); + + connectionProgressSpinner.setVisible(false); + sourceChoiceBox.getSelectionModel().selectedItemProperty().addListener(x -> changeVideoView()); + sourceChoiceBox.setTooltip(new Tooltip("If you have multiple cameras connected, enable \"Search for more videos\" in the settings view to see all available devices")); + + videoImagePlot.setData(dataSeries); + videoImagePlot.registerImageClickedHandler(this::onImageClicked); + videoImagePlot.registerImageClickedHandler(this::onSRM); + loadAvailableVideoDevicesAsync(); + + lineChart.getXAxis().setVisible(false); + lineChart.getYAxis().setVisible(false); + lineChart.setLegendVisible(false); + videoImagePlot.setLegendVisible(false); + lineChart.setMouseTransparent(true); + lineChart.setAnimated(false); + lineChart.setCreateSymbols(true); + lineChart.setAlternativeRowFillVisible(false); + lineChart.setAlternativeColumnFillVisible(false); + lineChart.setHorizontalGridLinesVisible(false); + lineChart.setVerticalGridLinesVisible(false); + lineChart.lookup(".chart-plot-background").setStyle("-fx-background-color: transparent;"); + lineChart.setData(lineDataSeries); + + ModeSelection.setValue("Single Point Mode"); + dataSeries.add(referencePoint); + + //State machine for the choice box for point mode selection + ModeSelection.getSelectionModel().selectedItemProperty().addListener((observable, oldValue, newValue) -> { + dataSeries.clear(); + if (Objects.equals(newValue, "Single Point Mode")) { + dataSeries.add(referencePoint); + PathControlPanel.setVisible(false); + lineDataSeries.clear(); + trackingPoint.setData(referencePoint.getData()); + } + if (Objects.equals(newValue, "Path Mode")) { + dataSeries.addAll(referencePointsListPath); + PathControlPanel.setVisible(true); + redrawPointsPathMode(); + trackingPoint = referencePointsListPath.get(0); + } + System.out.println("Selected item: " + newValue ); + // Add your custom code here... + }); + + clearAll.setOnAction((event) -> { + dataSeries.clear(); + referencePointsListPath.clear(); + lineDataSeries.clear(); + }); + } + + private void onSRM(double v, double v1) { + switch (ModeSelection.getValue()){ + case "Single Point Mode": + System.out.println("Set reference point to "+v+" "+v1); + referencePoint.setData(FXCollections.observableArrayList(new XYChart.Data<>(v,v1))); + trackingPoint.setData(referencePoint.getData()); + break; + case "Path Mode": + referencePointsListPath.add(new XYChart.Series<>("point",FXCollections.observableArrayList(new XYChart.Data<>(v,v1)))); + dataSeries.add(referencePointsListPath.get(referencePointsListPath.size()-1)); + //dataSeries.add(new XYChart.Series<>("Reference Funny",FXCollections.observableArrayList(new XYChart.Data<>(v,v1)))); + connectPointsPathMode(); + trackingPoint = new XYChart.Series<>("hi",FXCollections.observableArrayList(new XYChart.Data<>(referencePointsListPath.get(0).getData().get(0).getXValue(),referencePointsListPath.get(0).getData().get(0).getXValue()))); + + break; + } + + } + + private void connectPointsPathMode(){ + if(referencePointsListPath.size() == 1) { + return; + } + int i = referencePointsListPath.size() - 2; + + XYChart.Series series = referencePointsListPath.get(i); + XYChart.Series nextSeries = referencePointsListPath.get(i + 1); + + // Get the last point of the current series and the first point of the next series + XYChart.Data lastPoint = series.getData().get(series.getData().size() - 1); + XYChart.Data firstPointNext = nextSeries.getData().get(0); + + // Create a new series to represent the line between the two points + XYChart.Series lineSeries = new XYChart.Series<>(); + lineSeries.getData().add(new XYChart.Data<>(lastPoint.getXValue(), lastPoint.getYValue())); + lineSeries.getData().add(new XYChart.Data<>(firstPointNext.getXValue(), firstPointNext.getYValue())); + + // Add the line series to the line chart + lineDataSeries.add(lineSeries); + + } + private void redrawPointsPathMode(){ + for (int i = 0; i < referencePointsListPath.size() - 1; i++) { + XYChart.Series series = referencePointsListPath.get(i); + XYChart.Series nextSeries = referencePointsListPath.get(i + 1); + + // Get the last point of the current series and the first point of the next series + XYChart.Data lastPoint = series.getData().get(series.getData().size() - 1); + XYChart.Data firstPointNext = nextSeries.getData().get(0); + + // Create a new series to represent the line between the two points + XYChart.Series lineSeries = new XYChart.Series<>(); + lineSeries.getData().add(new XYChart.Data<>(lastPoint.getXValue(), lastPoint.getYValue())); + lineSeries.getData().add(new XYChart.Data<>(firstPointNext.getXValue(), firstPointNext.getYValue())); + + // Add the line series to the line chart + lineDataSeries.add(lineSeries); + } + } + + @Override + public void close() { + unregisterController(); + if (videoTimeline != null) { + videoTimeline.stop(); + } + imageDataManager.closeConnection(); + } + + /** + * Enables the Main View to inject the tracking data controller + * + */ + public void updateTrackingInformation() { + var selected = trackingService.getTrackingDataSource() != null && trackingService.getTimeline() != null; + trackingConnectedStatusBox.setSelected(selected); + } + + /** + * Enables the Main View to inject the status label at the bottom of the window + * + * @param statusLabel The injected label + */ + public void setStatusLabel(Label statusLabel) { + this.statusLabel = statusLabel; + this.statusLabel.setText(""); + } + + + /** + * Initializes the loading of available video devices. This is done asynchronously. + * While loading, the connection spinner shows. + */ + private void loadAvailableVideoDevicesAsync() { + connectionProgressSpinner.setVisible(true); + new Thread(() -> { + createDeviceIdMapping(userPreferencesGlobal.getBoolean("searchForMoreVideos", false)); + Platform.runLater(() -> { + sourceChoiceBox.getItems().addAll(deviceIdMapping.keySet()); + if (!deviceIdMapping.isEmpty()) { + sourceChoiceBox.getSelectionModel().select(0); + } else { + statusLabel.setText("No video devices found!"); + } + connectionProgressSpinner.setVisible(false); + }); + }).start(); + } + + /** + * Tests out available video device ids. All devices that don't throw an error are added to the list. + * This is bad style, but openCV does not offer to list available devices. + * @param exhaustiveSearch Whether all available devices shall be enumerated. If set to false, there's a minimal performance gain. + */ + private void createDeviceIdMapping(boolean exhaustiveSearch) { + if(!exhaustiveSearch){ + deviceIdMapping.put("Default Camera",0); + return; + } + + int currentDevice = 0; + boolean deviceExists = imageDataManager.openConnection(VideoSource.LIVESTREAM, currentDevice); + imageDataManager.closeConnection(); + while (deviceExists) { + deviceIdMapping.put("Camera " + currentDevice, currentDevice); + currentDevice++; + deviceExists = imageDataManager.openConnection(VideoSource.LIVESTREAM, currentDevice); + imageDataManager.closeConnection(); + } + } + + /** + * Changes the input stream for the video view. Also starts the timeline to update the current image. + */ + private void changeVideoView() { + if (!sourceChoiceBox.getSelectionModel().isEmpty()) { + var selectedItem = sourceChoiceBox.getSelectionModel().getSelectedItem(); + int deviceId = deviceIdMapping.get(selectedItem); + imageDataManager.closeConnection(); + imageDataManager.openConnection(VideoSource.LIVESTREAM, 2); + if (videoTimeline == null) { + videoTimeline = new Timeline(); + videoTimeline.setCycleCount(Animation.INDEFINITE); + videoTimeline.getKeyFrames().add( + new KeyFrame(Duration.millis(100), + event -> this.updateVideoImage()) + ); + videoTimeline.play(); + } + } else { + videoTimeline.stop(); + videoTimeline = null; + } + } + + /** + * Loads and displays the next image from the stream. + * If a measurement is scheduled, it queries the current tracking data and saves the data. + */ + private void updateVideoImage() { + var matrix = imageDataManager.readMat(); + if(matrix != null && !matrix.empty()) { + // Currently, we don't do image transformations, only tracking transformations + // matrix = applyImageTransformations(matrix); + currentShowingImage = ImageDataProcessor.Mat2BufferedImage(matrix); + } + + // Show Tracking Data + if(trackingConnectedStatusBox.isSelected()){ + updateTrackingData(); + } + + if(matrix != null && !matrix.empty()) { + videoImagePlot.setImage(ImageDataProcessor.Mat2Image(matrix, ".png")); + ((NumberAxis) lineChart.getXAxis()).setAutoRanging(false); + ((NumberAxis) lineChart.getXAxis()).setLowerBound(((NumberAxis) videoImagePlot.getXAxis()).getLowerBound()); + ((NumberAxis) lineChart.getXAxis()).setUpperBound(((NumberAxis) videoImagePlot.getXAxis()).getUpperBound()); + ((NumberAxis) lineChart.getXAxis()).setTickUnit(((NumberAxis) videoImagePlot.getXAxis()).getTickUnit()); + ((NumberAxis) lineChart.getYAxis()).setAutoRanging(false); + ((NumberAxis) lineChart.getYAxis()).setLowerBound(((NumberAxis) videoImagePlot.getYAxis()).getLowerBound()); + ((NumberAxis) lineChart.getYAxis()).setUpperBound(((NumberAxis) videoImagePlot.getYAxis()).getUpperBound()); + ((NumberAxis) lineChart.getYAxis()).setTickUnit(((NumberAxis) videoImagePlot.getYAxis()).getTickUnit()); + + lineChart.prefWidthProperty().bind(videoImagePlot.widthProperty()); + lineChart.prefHeightProperty().bind(videoImagePlot.heightProperty()); + lineChart.minWidthProperty().bind(videoImagePlot.widthProperty()); + lineChart.minHeightProperty().bind(videoImagePlot.heightProperty()); + lineChart.maxWidthProperty().bind(videoImagePlot.widthProperty()); + lineChart.maxHeightProperty().bind(videoImagePlot.heightProperty()); + } + } + + /** + * Loads the next tracking data point and displays it on the image-plot + */ + private void updateTrackingData(){ + var source = trackingService.getTrackingDataSource(); + var service = trackingService.getDataService(); + if(source == null || service == null){return;} + + source.update(); + List tools = service.loadNextData(1); + + if (tools.isEmpty()) return; + + lastTrackingData.clear(); + for (int i = 0; i < tools.size(); i++) { + Tool tool = tools.get(i); + if (dataSeries.size() <= i + 1) { + var series = new XYChart.Series(); + series.setName(tool.getName()); + series.getData().add(new XYChart.Data<>(0,0)); // Workaround to display legend + dataSeries.add(series); + series.getData().remove(0); + // TODO: The sensor curve needs reworking (apply on transformed data and dont shrink) +// videoImagePlot.initSensorCurve(series); + + + } + + if (lineDataSeries.size() <= i) { + // create series for line + // this series will include the reference point and current point + var lineSeries = new XYChart.Series(); + lineDataSeries.add(lineSeries); + lineSeries.getData().remove(0); + } + + + var series = dataSeries.get(i + 1); + var measurements = tool.getMeasurement(); + var point = measurements.get(measurements.size() - 1).getPos(); + var data = series.getData(); + var max_num_points = 6; // 1 + + var lineSeries = lineDataSeries.get(i); + //var lineData = lineSeries.getData(); + ObservableList> lineData = FXCollections.observableArrayList(); + var shifted_points = applyTrackingTransformation2d(point.getX(), point.getY(), point.getZ()); + var x_normalized = shifted_points[0] / currentShowingImage.getWidth(); + var y_normalized = shifted_points[1] / currentShowingImage.getHeight(); + lastTrackingData.add(new ExportMeasurement(tool.getName(), point.getX(), point.getY(), point.getZ(), shifted_points[0], shifted_points[1], shifted_points[2], x_normalized, y_normalized)); + + + data.add(new XYChart.Data<>(shifted_points[0], shifted_points[1])); + + lineData.add(new XYChart.Data<>(shifted_points[0], shifted_points[1])); + lineData.add(new XYChart.Data<>(trackingPoint.getData().get(0).getXValue(), trackingPoint.getData().get(0).getYValue())); + + lineSeries.setData(lineData); + + if(data.size() > max_num_points){ + data.remove(0); + //lineData.remove(0); + } + } + } + + /** + * Applies the transformation matrix to the image + * @param mat The image to be transformed + * @return The transformed image + */ + private Mat applyImageTransformations(Mat mat){ +// Imgproc.warpAffine(mat, mat, transformationMatrix.getTranslationMat(), mat.size()); +// Imgproc.warpAffine(mat, mat, transformationMatrix.getRotationMat(), mat.size()); +// Imgproc.warpAffine(mat, mat, transformationMatrix.getScaleMat(), mat.size()); + + /* + var imagePoints = transformationMatrix.getImagePoints(); + var trackingPoints = transformationMatrix.getTrackingPoints(); + var outMat = new Mat(); + if(!imagePoints.empty() && !trackingPoints.empty()) { + //Mat srcPoints = Converters.vector_Point_to_Mat(imagePoints, CvType.CV_64F); + //Mat dstPoints = Converters.vector_Point_to_Mat(trackingPoints, CvType.CV_64F); + + var matrix = Imgproc.getPerspectiveTransform(imagePoints, trackingPoints); + //var matrix = Calib3d.findHomography(imagePoints, trackingPoints, Calib3d.RANSAC); + + Imgproc.warpPerspective(mat, outMat, matrix, new Size()); + mat = outMat; + mat = outMat; + Imgproc.warpAffine(mat, outMat, transformationMatrix.getScaleMat(), mat.size()); + mat = outMat; + Imgproc.warpAffine(mat, outMat, transformationMatrix.getTranslationMat(), mat.size()); + } + */ + + if(roiDirty){ + // Set noExecute to false if the Roi should be calculated (and thus, the image cropped) + matrixRoi = MatHelper.calculateRoi(mat, true); + roiDirty = false; + } + mat = mat.submat(matrixRoi[0],matrixRoi[1], matrixRoi[2],matrixRoi[3]); + return mat; + } + + /** + * Applies the (2D) transformation on a tracking point + * @param x X-Coordinate of the point + * @param y Y-Coordinate of the point + * @param z Z-Coordinate of the point - Ignored in the 2d version + * @return The transformed point as array of length 3 (xyz) + */ + private double[] applyTrackingTransformation2d(double x, double y, double z) { + // TODO: Cache matrix + if (cachedTransformMatrix == null){ + cachedTransformMatrix = transformationMatrix.getTransformMatOpenCvEstimated2d(); + } + var vector = new Mat(3,1, CvType.CV_64F); + + if(userPreferencesGlobal.getBoolean("verticalFieldGenerator", false)){ + vector.put(0,0,z); + }else{ + vector.put(0,0,x); + } + vector.put(1,0,y); + vector.put(2,0,1); + + var pos_star = new Mat(2,1,CvType.CV_64F); + Core.gemm(cachedTransformMatrix, vector,1, new Mat(),1,pos_star); + double[] out = new double[3]; + out[0] = pos_star.get(0,0)[0]; + out[1] = pos_star.get(1,0)[0]; + out[2] = 0; + return out; + } + + /** + * Called, when the user clicks on the live image. Used to get landmarks for transformation. Uses 0 for the image plane as default + * @param x X-Coordinate in the image + * @param y Y-Coordinate in the image + */ + private void onImageClicked(double x, double y){ + System.out.println("clicked on image"); + var trackingData = lastTrackingData; + if(trackingData.size() > 0 && clicked_image_points.size() < 4) { + // We also directly save the tracking-coordinates at this point. + System.out.println("Image: (" + String.format(Locale.ENGLISH, "%.2f", x) + "," + String.format(Locale.ENGLISH, "%.2f", y) + ",0)\nImage (Relative): (" + String.format(Locale.ENGLISH, "%.2f", x) + "," + String.format(Locale.ENGLISH, "%.2f", (currentShowingImage.getHeight() - y)) + ",0)"); + for (var measurement : trackingData) { + System.out.println("Tracker " + measurement.toolName + ": (" + String.format(Locale.ENGLISH, "%.2f", measurement.x_raw) + "," + String.format(Locale.ENGLISH, "%.2f", measurement.y_raw) + "," + String.format(Locale.ENGLISH, "%.2f", measurement.z_raw) + ")\n"); + } + + clicked_image_points.add(new Point3(x, y, 0.0)); + if(userPreferencesGlobal.getBoolean("verticalFieldGenerator", false)) { + clicked_tracker_points.add(new Point3(trackingData.get(0).z_raw, trackingData.get(0).y_raw, trackingData.get(0).x_raw)); + }else { + clicked_tracker_points.add(new Point3(trackingData.get(0).x_raw, trackingData.get(0).y_raw, trackingData.get(0).z_raw)); + } + + } + } +} diff --git a/src/main/java/controller/AnnotationController.java b/src/main/java/controller/AnnotationController.java new file mode 100644 index 00000000..b1de7a2d --- /dev/null +++ b/src/main/java/controller/AnnotationController.java @@ -0,0 +1,882 @@ +package controller; + +import javafx.scene.control.Label; +import javafx.animation.FadeTransition; +import javafx.event.ActionEvent; +import javafx.fxml.FXML; +import javafx.geometry.HPos; +import javafx.geometry.Insets; +import javafx.scene.Node; +import javafx.scene.Scene; +import javafx.scene.control.*; +import javafx.scene.image.Image; +import javafx.scene.image.ImageView; +import javafx.scene.input.MouseEvent; +import javafx.scene.layout.GridPane; +import javafx.scene.layout.HBox; +import javafx.scene.layout.Pane; +import javafx.scene.layout.VBox; +import javafx.scene.paint.Color; +import javafx.scene.shape.Circle; +import javafx.scene.shape.Rectangle; +import javafx.scene.text.Font; +import javafx.scene.text.FontWeight; +import javafx.scene.text.Text; +import javafx.scene.transform.Scale; +import javafx.stage.DirectoryChooser; +import javafx.stage.FileChooser; +import javafx.stage.Modality; +import javafx.stage.Stage; +import javafx.util.Duration; +import util.AnnotationData; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.PrintWriter; +import java.net.URL; +import java.util.*; +import java.util.stream.Collectors; + +/** + * The AnnotationController class manages the user interface for + * uploading, viewing, and annotating images in the application. + */ +public class AnnotationController implements Controller { + @FXML + public VBox uploadedImages; // Where the users see the files + @FXML + public Button uploadImagesButton; + @FXML + public ScrollPane selectedImagePane; + public Button ExportButtonAll; + public Button ExportButton; + public Button clearMarksButton; + public Button helpButton; + public Button noTipButton; + @FXML + private ImageView selectedImageView; + private ImageView currentSelectedImageView; + @FXML + private Pane annotationPane; + @FXML + private Label uploadedImagesCountLabel; + + private Rectangle annotatedRectangle; + private Circle middlePoint; + private boolean dragged = false; + private double annotationPointX, annotationPointY; + private final Set uploadedFilePaths = new HashSet<>(); + // Store the paths of the selected Image, so you can Export the data based on these keys + private final Set selectedFilePaths = new HashSet<>(); + private int currentImageIndex = 0; // Default to the first image + private final List imageList = new ArrayList<>(); // Store all loaded images + private final Set noTipImageUrls = new HashSet<>(); + + /** + * Initializes the controller then sets up event handlers and initial states. + * + * @param location The location used to resolve relative paths for the root object. + * @param resources The resources used to localize the root object. + */ + @Override + public void initialize(URL location, ResourceBundle resources) { + System.out.println("Initializing Controller"); + if (annotationPane != null) { + setupAnnotationHandlers(); + } + + uploadedImages.setFocusTraversable(true); + uploadedImages.setOnKeyPressed(event -> { + switch (event.getCode()) { + case RIGHT: + selectNextImage(); + break; + case LEFT: + selectPreviousImage(); + break; + case A: + if (event.isControlDown()) { + selectAllCheckboxes(); + } + break; + default: + break; + } + }); + } + + /** + * Selects all checkboxes for uploaded images. + */ + private void selectAllCheckboxes() { + for (Node node : uploadedImages.getChildren()) { + if (node instanceof HBox) { + CheckBox checkBox = (CheckBox) ((HBox) node).getChildren().get(1); + checkBox.setSelected(true); + } + } + } + + /** + * Sets up mouse event handlers for annotation actions. + */ + private void setupAnnotationHandlers() { + annotationPane.setOnMousePressed(event -> { + if (currentSelectedImageView != null && !noTipImageUrls.contains(currentSelectedImageView.getImage().getUrl())) { + pressedAnnotationEvent(event); + } + }); + annotationPane.setOnMouseDragged(event -> { + if (currentSelectedImageView != null && !noTipImageUrls.contains(currentSelectedImageView.getImage().getUrl())) { + dragAnnotationEvent(event); + } + }); + annotationPane.setOnMouseReleased(event -> { + if (currentSelectedImageView != null && !noTipImageUrls.contains(currentSelectedImageView.getImage().getUrl())) { + releasedAnnotationEvent(); + } + }); + } + + /** + * Selects the next image in the list. + * If no next image is available, it prints a message. + */ + public void selectNextImage() { + // Check if there's a next image in the list + if (currentImageIndex < imageList.size() - 1) { + currentImageIndex++; + Image nextImage = imageList.get(currentImageIndex); + ImageView nextImageView = findImageViewForImage(nextImage); + selectImage(nextImage, nextImageView); + } else { + System.out.println("No next image available."); + } + } + + /** + * Selects the previous image in the list. + * If no previous image is available, it prints a message. + */ + public void selectPreviousImage() { + // Check if there's a previous image in the list + if (currentImageIndex > 0) { + currentImageIndex--; + Image prevImage = imageList.get(currentImageIndex); + ImageView prevImageView = findImageViewForImage(prevImage); + selectImage(prevImage, prevImageView); + } else { + System.out.println("No previous image available."); + } + } + + /** + * Finds the ImageView of a given image. + * + * @param image The image to find the ImageView for. + * @return The ImageView of the image. + */ + private ImageView findImageViewForImage(Image image) { + for (Node node : uploadedImages.getChildren()) { + if (node instanceof HBox) { + ImageView imageView = (ImageView) ((HBox) node).getChildren().get(0); + if (imageView.getImage().equals(image)) { + return imageView; + } + } + } + return null; // Not found + } + + /** + * Closes the controller and cleans up. + */ + @FXML + @Override + public void close() { + unregisterController(); + } + + /** + * Handles the upload functionality for selecting and displaying images. + * + * @param actionEvent The event triggered by the upload button. + */ + @FXML + public void Handle_Upload_Functionality(ActionEvent actionEvent) { + try { + FileChooser fileChooser = new FileChooser(); + fileChooser.setTitle("Select Images"); + fileChooser.setInitialDirectory(new File(System.getProperty("user.home"))); + fileChooser.getExtensionFilters().add( + new FileChooser.ExtensionFilter("Image Files", "*.jpg", "*.png", "*.bmp", "*.gif", "*.jpeg", "*.tiff", "*.tif") + ); + Stage currentStage = (Stage) ((javafx.scene.Node) actionEvent.getSource()).getScene().getWindow(); + + List selectedImages = fileChooser.showOpenMultipleDialog(currentStage); + + if (selectedImages != null) { + List duplicateFiles = new ArrayList<>(); + for (File file : selectedImages) { + if (!uploadedFilePaths.contains(file.getAbsolutePath())) { + displayImage(file); + uploadedFilePaths.add(file.getAbsolutePath()); + } else { + duplicateFiles.add(file.getName()); + } + } + if (!duplicateFiles.isEmpty()) { + showDuplicateFilesAlert(duplicateFiles); + } + } + } catch (Exception e) { + System.err.println("Error while choosing File: " + e.getMessage()); + } + } + + /** + * Displays an alert for duplicate files. + * + * @param duplicateFiles List of duplicate files. + */ + private void showDuplicateFilesAlert(List duplicateFiles) { + Alert alert = new Alert(Alert.AlertType.INFORMATION); + alert.setTitle("Duplicate Files"); + alert.setHeaderText("The following files have already been uploaded:"); + VBox vbox = new VBox(5); + for (String fileName : duplicateFiles) { + Text fileText = new Text(fileName); + vbox.getChildren().add(fileText); + } + ScrollPane scrollPane = new ScrollPane(vbox); + scrollPane.setPrefSize(300, 150); + scrollPane.setVbarPolicy(ScrollPane.ScrollBarPolicy.AS_NEEDED); + scrollPane.setHbarPolicy(ScrollPane.ScrollBarPolicy.NEVER); + alert.getDialogPane().setContent(scrollPane); + alert.setResizable(true); + alert.showAndWait(); + } + + /** + * Displays an image file in the application. + * + * @param file The image file to be displayed. + */ + private void displayImage(File file) { + HBox hbox = new HBox(); + hbox.setSpacing(10); + + CheckBox checkBox = new CheckBox(); + checkBox.setSelected(false); + + HBox.setMargin(checkBox, new Insets(10, 30, 10, 10)); + + Image image = new Image(file.toURI().toString()); + imageList.add(image); // Adding the image to the list + ImageView imageView = new ImageView(image); + imageView.setFitHeight(100); + imageView.setFitWidth(100); + imageView.setPreserveRatio(true); + + imageView.setOnMouseClicked(event -> { + selectImage(image, imageView); + }); + // Add or Remove the selected file paths to the HashSet + checkBox.selectedProperty().addListener((observable, oldValue, newValue) -> { + if (newValue) { // Checkbox is now selected + selectedFilePaths.add(imageView.getImage().getUrl()); + } else { // Checkbox is now unselected + selectedFilePaths.remove(imageView.getImage().getUrl()); + } + }); + + hbox.getChildren().add(imageView); + hbox.getChildren().add(checkBox); + uploadedImages.getChildren().add(hbox); + updateUploadedImagesCount(); + } + + /** + * Selects an image for display and annotation. + * Updates image view, also handles scrolling and zooming functionalities. + * + * @param image The image to be selected. + * @param imageView The ImageView of the selected image. + */ + private void selectImage(Image image, ImageView imageView) { + try { + if (imageList.isEmpty()) { + selectedImageView.setImage(null); + System.out.println("No images to display."); + return; + } + currentImageIndex = imageList.indexOf(image); + if (currentImageIndex == -1) { + System.out.println("Selected image is not in the image list."); + return; + } + if (selectedImageView == null) { + System.out.println("Error: selectedImageView is not initialized."); + return; + } + selectedImageView.setImage(image); + selectedImageView.setFitWidth(selectedImageView.getScene().getWidth()); + selectedImageView.setPreserveRatio(true); + if (currentSelectedImageView != null && currentSelectedImageView != imageView) { + currentSelectedImageView.setStyle(""); + } + + if (imageView != null) { + currentSelectedImageView = imageView; + imageView.setStyle("-fx-effect: dropshadow(three-pass-box, deepskyblue, 10, 0, 0, 0); -fx-border-color: blue; -fx-border-width: 2;"); + } + scrollToSelectedImage(); + if (annotationPane != null) { + annotationPane.getTransforms().clear(); + Scale scale = new Scale(); + annotationPane.getTransforms().add(scale); + annotationPane.setOnScroll(event -> { + if (event.isControlDown()) { + double zoomFactor = 1.05; + scale.setPivotX(event.getX()); + scale.setPivotY(event.getY()); + + if (event.getDeltaY() > 0) { + scale.setX(scale.getX() * zoomFactor); + scale.setY(scale.getY() * zoomFactor); + } else if (scale.getX() > 1.0 && scale.getY() > 1.0) { + scale.setX(scale.getX() / zoomFactor); + scale.setY(scale.getY() / zoomFactor); + } + event.consume(); + } + }); + } + checkForExistingAnnotationData(); + } catch (Exception e) { + System.out.println("Exception in selectImage: " + e.getMessage()); + } + } + + /** + * Scrolls the selected image into view within the selectedImagePane. + * Also adjusts the scrollbar. + */ + private void scrollToSelectedImage() { + if (currentSelectedImageView != null && selectedImagePane != null) { + double viewportHeight = selectedImagePane.getHeight(); + double imageY = currentSelectedImageView.localToScene(currentSelectedImageView.getBoundsInLocal()).getMinY(); + double offsetY = imageY - selectedImagePane.getScene().getY() - viewportHeight / 2 + currentSelectedImageView.getBoundsInLocal().getHeight() / 2; + double vValue = offsetY / (uploadedImages.getHeight() - viewportHeight); + selectedImagePane.setVvalue(Math.max(0, Math.min(vValue, 1))); // Clamp vValue to be between 0 and 1 + } + } + + /** + * Removes any existing annotations from pane. + * Also deletes the annotation data of the selected image. + */ + public void clearAnnotations() { + if (annotatedRectangle != null) { + annotationPane.getChildren().remove(annotatedRectangle); + annotationPane.getChildren().remove(middlePoint); + AnnotationData.getInstance().deleteAnnotation(selectedImageView.getImage().getUrl()); + middlePoint = null; + annotatedRectangle = null; + } + } + + /** + * Clears any existing annotations, then retrieves and displays annotation data. + * Also creates and displays the middle point of the annotation. + */ + private void checkForExistingAnnotationData() { + // Clear existing annotations if any + if (annotatedRectangle != null) { + annotationPane.getChildren().remove(annotatedRectangle); + annotatedRectangle = null; + } + if (middlePoint != null) { + annotationPane.getChildren().remove(middlePoint); + middlePoint = null; + } + + String imageUrl = selectedImageView.getImage().getUrl(); + if (noTipImageUrls.contains(imageUrl)) { + showAlert("Info", "This image is marked as 'No Tip' and cannot be annotated."); + return; + } + + // Reload annotation if it exists + annotatedRectangle = AnnotationData.getInstance().getAnnotation(imageUrl); + if (annotatedRectangle != null) { + // Apply the transformations to the rectangle to match the image dimensions + annotatedRectangle.setX(annotatedRectangle.getX() * selectedImageView.getImage().getWidth()); + annotatedRectangle.setY(annotatedRectangle.getY() * selectedImageView.getImage().getHeight()); + annotatedRectangle.setWidth(annotatedRectangle.getWidth() * selectedImageView.getImage().getWidth()); + annotatedRectangle.setHeight(annotatedRectangle.getHeight() * selectedImageView.getImage().getHeight()); + + annotationPane.getChildren().add(annotatedRectangle); + + middlePoint = new Circle(0, 0, 2); + middlePoint.setFill(Color.rgb(6, 207, 236)); + middlePoint.setCenterX(annotatedRectangle.getX() + (annotatedRectangle.getWidth() / 2)); + middlePoint.setCenterY(annotatedRectangle.getY() + (annotatedRectangle.getHeight() / 2)); + annotationPane.getChildren().add(middlePoint); + } + } + + /** + * Allows for a variable annotation rectangle to be displayed at the cursor position. + * Only works when control is held down. + */ + private void dragAnnotationEvent(MouseEvent event) { + if (event.isControlDown()) { + double x2 = event.getX(); + double y2 = event.getY(); + annotatedRectangle.setX(annotationPointX); + annotatedRectangle.setY(annotationPointY); + annotatedRectangle.setWidth(Math.abs(x2 - annotationPointX)); + annotatedRectangle.setHeight(Math.abs(y2 - annotationPointY)); + middlePoint.setCenterX(annotationPointX + annotatedRectangle.getWidth() / 2); + middlePoint.setCenterY(annotationPointY + annotatedRectangle.getHeight() / 2); + dragged = true; + } + } + + /** + * Allows for a square annotation rectangle to be displayed at the cursor position. + * Default size for annotations. + */ + private void pressedAnnotationEvent(MouseEvent event) { + annotationPointX = event.getX(); + annotationPointY = event.getY(); + + if (annotatedRectangle == null) { + annotatedRectangle = new Rectangle(); + annotatedRectangle.setFill(Color.TRANSPARENT); + annotatedRectangle.setStroke(Color.rgb(6, 207, 236)); + annotatedRectangle.setStrokeWidth(2); + annotatedRectangle.setVisible(true); + annotationPane.getChildren().add(annotatedRectangle); // Add it to the Scene + } + if (middlePoint == null) { + middlePoint = new Circle(0, 0, 2); + middlePoint.setFill(Color.rgb(6, 207, 236)); + annotationPane.getChildren().add(middlePoint); + } + } + + /** + * Handles the released annotation event. + * Finalizes the dragged annotation and stores annotation data. + */ + private void releasedAnnotationEvent() { + if (!dragged) { + annotatedRectangle.setX(annotationPointX - 10); + annotatedRectangle.setY(annotationPointY - 10); + annotatedRectangle.setWidth(20); + annotatedRectangle.setHeight(20); + middlePoint.setCenterX(annotationPointX); + middlePoint.setCenterY(annotationPointY); + } else { + // Calculate the Middle point of the rectangle + annotationPointX += annotatedRectangle.getWidth() / 2; + annotationPointY += annotatedRectangle.getHeight() / 2; + } + + dragged = false; + + AnnotationData.getInstance().addAnnotation( + selectedImageView.getImage().getUrl(), + annotationPointX / selectedImageView.getImage().getWidth(), + annotationPointY / selectedImageView.getImage().getHeight(), + annotatedRectangle.getWidth() / selectedImageView.getImage().getWidth(), + annotatedRectangle.getHeight() / selectedImageView.getImage().getHeight() + ); + } + + /** + * Handles the export action for saving annotations of selected images to files. + * Allows for annotation data to be saved as text files in the desired directory. + * Does not allow for unannotated images to be exported. + * + * @param event The ActionEvent representing the export action. + */ + @FXML + private void handleExportAction(ActionEvent event) { + if (uploadedImages.getChildren().isEmpty()) { + showAlert("Export Error", "There are no images to export."); + return; + } + Set selectedImagesUrls = uploadedImages.getChildren().stream() + .filter(node -> node instanceof HBox) + .map(node -> (HBox) node) + .filter(hbox -> ((CheckBox) hbox.getChildren().get(1)).isSelected()) + .map(hbox -> (ImageView) hbox.getChildren().get(0)) + .map(ImageView::getImage) + .map(Image::getUrl) + .collect(Collectors.toSet()); + + if (selectedImagesUrls.isEmpty()) { + showAlert("No Selection", "No images have been check-marked for export."); + return; + } + + // Check if all selected images have annotations + List unannotatedSelectedImages = selectedImagesUrls.stream() + .filter(url -> !AnnotationData.getInstance().getAnnotations().containsKey(url)) + .map(url -> new File(url).getName()) + .collect(Collectors.toList()); + + if (!unannotatedSelectedImages.isEmpty()) { + showUnannotatedImagesAlert(unannotatedSelectedImages); + return; + } + + Map annotations = AnnotationData.getInstance().getAnnotations(); + DirectoryChooser directoryChooser = new DirectoryChooser(); + directoryChooser.setTitle("Select Directory to Save Annotations"); + File selectedDirectory = directoryChooser.showDialog(((Node) event.getSource()).getScene().getWindow()); + if (selectedDirectory != null) { + selectedImagesUrls.forEach(url -> { + try { + File file = new File(new URL(url).toURI()); + String fileName = file.getName().substring(0, file.getName().lastIndexOf('.')) + ".txt"; + File annotationFile = new File(selectedDirectory, fileName); + saveAnnotationsToFile(annotationFile, annotations.get(url)); + } catch (Exception e) { + showAlert("Error", "Failed to save annotations for " + url); + } + }); + } + } + + /** + * Handles the export action for saving annotations of all displayed images to files. + * Allows for annotation data to be saved as text files in the desired directory. + * Does not allow for unannotated images to be exported. + * + * @param event The ActionEvent representing the export all action. + */ + @FXML + private void handleExportAllAction(ActionEvent event) { + if (uploadedImages.getChildren().isEmpty()) { + showAlert("Export Error", "There are no images to export."); + return; + } + Set allImagesUrls = uploadedImages.getChildren().stream() + .filter(node -> node instanceof HBox) + .map(node -> (HBox) node) + .map(hbox -> (ImageView) hbox.getChildren().get(0)) + .map(ImageView::getImage) + .map(Image::getUrl) + .collect(Collectors.toSet()); + + // Check if all uploaded images have annotations + List unannotatedImages = allImagesUrls.stream() + .filter(url -> !AnnotationData.getInstance().getAnnotations().containsKey(url)) + .map(url -> new File(url).getName()) + .collect(Collectors.toList()); + + if (!unannotatedImages.isEmpty()) { + showUnannotatedImagesAlert(unannotatedImages); + return; + } + + DirectoryChooser directoryChooser = new DirectoryChooser(); + directoryChooser.setTitle("Select Directory to Save Annotations"); + File selectedDirectory = directoryChooser.showDialog(((Node) event.getSource()).getScene().getWindow()); + if (selectedDirectory != null) { + AnnotationData.getInstance().getAnnotations().entrySet().stream() + .filter(entry -> allImagesUrls.contains(entry.getKey())) + .forEach(entry -> { + try { + String path = entry.getKey(); + File file = new File(new URL(path).toURI()); + String fileName = file.getName().substring(0, file.getName().lastIndexOf('.')) + ".txt"; + File annotationFile = new File(selectedDirectory, fileName); + saveAnnotationsToFile(annotationFile, entry.getValue()); + } catch (Exception e) { + showAlert("Error", "Failed to save annotations for " + entry.getKey()); + } + }); + } + } + + /** + * Handles the alert that unannotated images cannot be exported. + * + * @param unannotatedImages A list of the names of unannotated images. + */ + private void showUnannotatedImagesAlert(List unannotatedImages) { + Alert alert = new Alert(Alert.AlertType.INFORMATION); + alert.setTitle("Unannotated Images"); + alert.setHeaderText("The following images have no annotations:"); + VBox vbox = new VBox(5); + for (String imageName : unannotatedImages) { + Text imageText = new Text(imageName); + vbox.getChildren().add(imageText); + } + ScrollPane scrollPane = new ScrollPane(vbox); + scrollPane.setPrefSize(300, 150); + scrollPane.setVbarPolicy(ScrollPane.ScrollBarPolicy.AS_NEEDED); + scrollPane.setHbarPolicy(ScrollPane.ScrollBarPolicy.NEVER); + alert.getDialogPane().setContent(scrollPane); + alert.setResizable(true); + alert.showAndWait(); + } + + /** + * Displays a generic information alert. + * + * @param title The title of the alert. + * @param content The content of the alert. + */ + private void showAlert(String title, String content) { + Alert alert = new Alert(Alert.AlertType.INFORMATION); + alert.setTitle(title); + alert.setHeaderText(null); + alert.setContentText(content); + alert.showAndWait(); + } + + /** + * Handles the Saving of Annotations to a File. + * + * @param file The file to save the annotations to. + * @param annotation The annotation data to save. + */ + private void saveAnnotationsToFile(File file, AnnotationData.PublicAnnotation annotation) { + try (PrintWriter writer = new PrintWriter(file)) { + String line = String.format("%d %.5f %.5f %.5f %.5f", + 0, annotation.getMiddlePointX(), annotation.getMiddlePointY(), + annotation.getBoundingBoxWidth(), annotation.getBoundingBoxHeight()); + writer.println(line); + } catch (FileNotFoundException e) { + System.err.println("Error while saving Annotation to File: " + e.getMessage()); + } + } + + /** + * Handles the deletion functionality for selected images. + * Removes related image data and annotation data from all locations. + * Updates currently displayed image and image status. + */ + @FXML + public void deletionfunctionality() { + List toRemove = new ArrayList<>(); + List imagesToRemove = new ArrayList<>(); + boolean currentDisplayedRemoved = false; + boolean atLeastOneSelected = false; + + for (Node node : uploadedImages.getChildren()) { + if (node instanceof HBox hbox) { + ImageView imageView = (ImageView) hbox.getChildren().get(0); + CheckBox checkBox = (CheckBox) hbox.getChildren().get(1); + if (checkBox.isSelected()) { + atLeastOneSelected = true; + String imagePath = null; + try { + imagePath = new File(new URL(imageView.getImage().getUrl()).toURI()).getAbsolutePath(); + imagesToRemove.add(imageView.getImage()); + toRemove.add(node); + uploadedFilePaths.remove(imagePath); + selectedFilePaths.remove(imagePath); + noTipImageUrls.remove(imageView.getImage().getUrl()); // Remove from noTipImageUrls set + AnnotationData.getInstance().deleteAnnotation(imageView.getImage().getUrl()); // Remove annotation + + if (imageView.equals(currentSelectedImageView)) { + currentDisplayedRemoved = true; + } + } catch (Exception e) { + showAlert("Error", "Could not retrieve file path from the image: " + e.getMessage()); + System.err.println("Error while retrieving filepath during deletion: " + e.getMessage()); + } + } + } + } + + if (atLeastOneSelected) { + uploadedImages.getChildren().removeAll(toRemove); + imageList.removeAll(imagesToRemove); + if (currentDisplayedRemoved) { + selectedImageView.setImage(null); + currentSelectedImageView = null; + currentImageIndex = Math.min(currentImageIndex, imageList.size() - 1); + } + updateUploadedImagesCount(); + } else { + showAlert("Notice", "No images selected for deletion."); + } + + // Notify if all images have been deleted + if (uploadedImages.getChildren().isEmpty()) { + showAlert("Notice", "All images have been deleted."); + } + + clearAnnotations(); + } + + /** + * Updates the count of uploaded images. + */ + private void updateUploadedImagesCount() { + int count = uploadedImages.getChildren().size(); + uploadedImagesCountLabel.setText("Images: " + count); + } + + /** + * Handles the action when the help button is clicked. + * Creates window with information on how to use this application. + */ + @FXML + public void handleHelpButtonAction() { + try { + Stage helpStage = new Stage(); + helpStage.initModality(Modality.APPLICATION_MODAL); + helpStage.setTitle("Help"); + VBox helpContent = new VBox(); + helpContent.setPadding(new Insets(10)); + helpContent.setSpacing(10); + helpContent.setStyle("-fx-background-color: #f0f0f0; -fx-border-radius: 10; -fx-background-radius: 10;"); + GridPane helpGrid = getHelpGrid(); + helpGrid.setPadding(new Insets(10)); + helpGrid.setVgap(10); + helpGrid.setHgap(10); + helpGrid.setStyle("-fx-background-color: #ffffff; -fx-border-radius: 10; -fx-background-radius: 10; -fx-border-color: #cccccc; -fx-border-width: 1;"); + helpContent.getChildren().add(helpGrid); + Scene helpScene = new Scene(helpContent); + FadeTransition fadeIn = new FadeTransition(Duration.millis(500), helpContent); + fadeIn.setFromValue(0.0); + fadeIn.setToValue(1.0); + fadeIn.play(); + helpStage.setScene(helpScene); + helpStage.sizeToScene(); + helpStage.showAndWait(); + } catch (Exception e) { + System.err.println("Error while opening Help window: " + e.getMessage()); + } + } + + /** + * Generates a grid containing helpful information. + * + * @return A GridPane containing helpful information. + */ + private static GridPane getHelpGrid() { + String[][] helpTextArray = { + {"Import", "Click this button to import images into the tool."}, + {"Clear Marks", "Click this button to clear all annotation marks from the currently displayed image."}, + {"Delete", "Click this button to delete the selected images from the tool."}, + {"Specific Export", "Click this button to export annotations for the selected images."}, + {"Export All", "Click this button to export annotations for all images."}, + {"No Tip", "Click this button to rename selected images to be marked as NoTip found. Now the picture "}, + { "", "won't be allowed to be annotated, unless you reload it in the application again."}, + {"Next", "Click this button to navigate to the next image."}, + {"Previous", "Click this button to navigate to the previous image."}, + {"", "To annotate an image, click and drag to create a bounding box. Hold down the Ctrl key to resize the bounding box."}, + {"Special Keybindings:", ""}, + {"Ctrl + Mousewheel (on an image): ", "Zoom"}, + {"Ctrl + Mouse drag (on an image): ", "Custom sized annotation box"}, + {"Ctrl + A (After Selecting one image Checkbox): ", "All pictures will be check marked"}, + {"Arrows: ", "Move zoomed images"} + }; + GridPane gridPane = new GridPane(); + gridPane.setPadding(new Insets(10)); + gridPane.setVgap(10); + gridPane.setHgap(10); + + for (int i = 0; i < helpTextArray.length; i++) { + String[] helpEntry = helpTextArray[i]; + Text buttonName = new Text(helpEntry[0]); + buttonName.setFont(Font.font("Arial", FontWeight.BOLD, 16)); + GridPane.setHalignment(buttonName, HPos.LEFT); + Text description = new Text(helpEntry[1]); + description.setFont(Font.font("Arial", 16)); + GridPane.setHalignment(description, HPos.LEFT); + gridPane.add(buttonName, 0, i); + gridPane.add(description, 1, i); + } + + return gridPane; + } + + /** + * Handles the action to mark images as No Tip. + */ + @FXML + private void handleNoTipAction() { + List selectedImageUrls = new ArrayList<>(); + + for (Node node : uploadedImages.getChildren()) { + if (node instanceof HBox hbox) { + CheckBox checkBox = (CheckBox) hbox.getChildren().get(1); + if (checkBox.isSelected()) { + ImageView imageView = (ImageView) hbox.getChildren().get(0); + selectedImageUrls.add(imageView.getImage().getUrl()); + } + } + } + if (selectedImageUrls.isEmpty()) { + showAlert("No Selection", "No images have been check-marked for No Tip processing."); + return; + } + + // Confirmation dialog + Alert confirmationAlert = new Alert(Alert.AlertType.CONFIRMATION); + confirmationAlert.setTitle("Confirmation"); + confirmationAlert.setHeaderText("Mark as No Tip"); + confirmationAlert.setContentText("Are you sure you want to mark these pictures as No Tip Found?"); + Optional result = confirmationAlert.showAndWait(); + if (result.isPresent() && result.get() != ButtonType.OK) { + return; // If not confirmed, exit the method + } + + boolean allMarked = true; + StringBuilder successMessage = new StringBuilder("The following files have been marked as No Tip Found in the application:\n"); + for (String imageUrl : selectedImageUrls) { + try { + markImageAsNoTip(imageUrl); + successMessage.append(imageUrl).append("\n"); + } catch (Exception e) { + allMarked = false; + // showAlert("Error", "An error occurred: " + e.getMessage()); + } + } + if (allMarked) { + showAlert("Success", successMessage.toString()); + } + } + + /** + * Marks an image as No Tip. + * + * @param imageUrl The URL of the image to be marked as No Tip. + */ + private void markImageAsNoTip(String imageUrl) { + // Add "No Tip" annotation + AnnotationData.getInstance().addAnnotation( + imageUrl, 0, 0, 0, 0); + + // Add to No Tip set + noTipImageUrls.add(imageUrl); + + // Update the UI + for (Node node : uploadedImages.getChildren()) { + if (node instanceof HBox) { + ImageView imageView = (ImageView) ((HBox) node).getChildren().get(0); + if (imageView.getImage().getUrl().equals(imageUrl)) { + ((CheckBox) ((HBox) node).getChildren().get(1)).setText("No Tip"); + break; + } + } + } + + if (currentSelectedImageView != null && currentSelectedImageView.getImage().getUrl().equals(imageUrl)) { + selectedImageView.setImage(null); + currentSelectedImageView = null; + currentImageIndex = Math.min(currentImageIndex, imageList.size() - 1); + } + clearAnnotations(); + } +} diff --git a/src/main/java/controller/AutoTrackController.java b/src/main/java/controller/AutoTrackController.java index 05789c4e..7a884038 100644 --- a/src/main/java/controller/AutoTrackController.java +++ b/src/main/java/controller/AutoTrackController.java @@ -14,6 +14,8 @@ import javafx.collections.FXCollections; import javafx.collections.ObservableList; import javafx.fxml.FXML; +import javafx.scene.chart.LineChart; +import javafx.scene.chart.NumberAxis; import javafx.scene.chart.XYChart; import javafx.scene.control.Button; import javafx.scene.control.Label; @@ -39,6 +41,7 @@ import java.nio.file.Path; import java.util.List; import java.util.*; +import java.util.logging.Level; import java.util.logging.Logger; import java.util.prefs.Preferences; @@ -80,6 +83,10 @@ public class AutoTrackController implements Controller { public PlottableImage videoImagePlot; @FXML public CheckBox use3dTransformCheckBox; + @FXML + public CheckBox inSetReferenceMode; + @FXML + public LineChart lineChart; private TrackingDataController trackingDataController; private Label statusLabel; @@ -102,6 +109,13 @@ public class AutoTrackController implements Controller { private final ObservableList clicked_tracker_points = FXCollections.observableArrayList(); private Mat cachedTransformMatrix = null; + private final XYChart.Series referencePoint = new XYChart.Series(); + private final ObservableList> lineDataSeries = FXCollections.observableArrayList(); + + NumberAxis xAxis = new NumberAxis(-500, 500, 100); + NumberAxis yAxis = new NumberAxis(-500, 500, 100); + + @Override public void initialize(URL location, ResourceBundle resources) { registerController(); @@ -122,10 +136,35 @@ public void initialize(URL location, ResourceBundle resources) { generateMatrixButton.disableProperty().bind(Bindings.size(clicked_image_points).lessThan(4)); generateMatrixButton.textProperty().bind(Bindings.concat("Generate (",Bindings.size(clicked_image_points),"/4)")); + referencePoint.setName("reference point"); + referencePoint.getData().add(new XYChart.Data<>(100,100)); + dataSeries.add(referencePoint); + videoImagePlot.setData(dataSeries); videoImagePlot.registerImageClickedHandler(this::onImageClicked); - + videoImagePlot.registerImageClickedHandler(this::onSRM); loadAvailableVideoDevicesAsync(); + + lineChart.getXAxis().setVisible(false); + lineChart.getYAxis().setVisible(false); + lineChart.setLegendVisible(false); + videoImagePlot.setLegendVisible(false); + lineChart.setMouseTransparent(true); + lineChart.setAnimated(false); + lineChart.setCreateSymbols(true); + lineChart.setAlternativeRowFillVisible(false); + lineChart.setAlternativeColumnFillVisible(false); + lineChart.setHorizontalGridLinesVisible(false); + lineChart.setVerticalGridLinesVisible(false); + lineChart.lookup(".chart-plot-background").setStyle("-fx-background-color: transparent;"); + lineChart.setData(lineDataSeries); + } + + private void onSRM(double v, double v1) { + if (inSetReferenceMode.isSelected()) { + System.out.println("Set reference point to "+v+" "+v1); + referencePoint.setData(FXCollections.observableArrayList(new XYChart.Data<>(v,v1))); + } } @Override @@ -210,7 +249,7 @@ private void changeVideoView() { var selectedItem = sourceChoiceBox.getSelectionModel().getSelectedItem(); int deviceId = deviceIdMapping.get(selectedItem); imageDataManager.closeConnection(); - imageDataManager.openConnection(VideoSource.LIVESTREAM, deviceId); + imageDataManager.openConnection(VideoSource.LIVESTREAM, 1); if (videoTimeline == null) { videoTimeline = new Timeline(); videoTimeline.setCycleCount(Animation.INDEFINITE); @@ -249,6 +288,21 @@ private void updateVideoImage() { } if(matrix != null && !matrix.empty()) { videoImagePlot.setImage(ImageDataProcessor.Mat2Image(matrix, ".png")); + ((NumberAxis) lineChart.getXAxis()).setAutoRanging(false); + ((NumberAxis) lineChart.getXAxis()).setLowerBound(((NumberAxis) videoImagePlot.getXAxis()).getLowerBound()); + ((NumberAxis) lineChart.getXAxis()).setUpperBound(((NumberAxis) videoImagePlot.getXAxis()).getUpperBound()); + ((NumberAxis) lineChart.getXAxis()).setTickUnit(((NumberAxis) videoImagePlot.getXAxis()).getTickUnit()); + ((NumberAxis) lineChart.getYAxis()).setAutoRanging(false); + ((NumberAxis) lineChart.getYAxis()).setLowerBound(((NumberAxis) videoImagePlot.getYAxis()).getLowerBound()); + ((NumberAxis) lineChart.getYAxis()).setUpperBound(((NumberAxis) videoImagePlot.getYAxis()).getUpperBound()); + ((NumberAxis) lineChart.getYAxis()).setTickUnit(((NumberAxis) videoImagePlot.getYAxis()).getTickUnit()); + + lineChart.prefWidthProperty().bind(videoImagePlot.widthProperty()); + lineChart.prefHeightProperty().bind(videoImagePlot.heightProperty()); + lineChart.minWidthProperty().bind(videoImagePlot.widthProperty()); + lineChart.minHeightProperty().bind(videoImagePlot.heightProperty()); + lineChart.maxWidthProperty().bind(videoImagePlot.widthProperty()); + lineChart.maxHeightProperty().bind(videoImagePlot.heightProperty()); } } @@ -268,7 +322,7 @@ private void updateTrackingData(){ lastTrackingData.clear(); for (int i = 0; i < tools.size(); i++) { Tool tool = tools.get(i); - if (dataSeries.size() <= i) { + if (dataSeries.size() <= i + 1) { var series = new XYChart.Series(); series.setName(tool.getName()); series.getData().add(new XYChart.Data<>(0,0)); // Workaround to display legend @@ -276,23 +330,44 @@ private void updateTrackingData(){ series.getData().remove(0); // TODO: The sensor curve needs reworking (apply on transformed data and dont shrink) // videoImagePlot.initSensorCurve(series); + + + } + + if (lineDataSeries.size() <= i) { + // create series for line + // this series will include the reference point and current point + var lineSeries = new XYChart.Series(); + lineDataSeries.add(lineSeries); + lineSeries.getData().remove(0); } - var series = dataSeries.get(i); + + var series = dataSeries.get(i + 1); var measurements = tool.getMeasurement(); var point = measurements.get(measurements.size() - 1).getPos(); var data = series.getData(); - var max_num_points = 4; // 1 + var max_num_points = 6; // 1 + var lineSeries = lineDataSeries.get(i); + //var lineData = lineSeries.getData(); + ObservableList> lineData = FXCollections.observableArrayList(); var shifted_points = use3dTransformCheckBox.isSelected() ? applyTrackingTransformation3d(point.getX(), point.getY(), point.getZ()) : applyTrackingTransformation2d(point.getX(), point.getY(), point.getZ()); var x_normalized = shifted_points[0] / currentShowingImage.getWidth(); var y_normalized = shifted_points[1] / currentShowingImage.getHeight(); lastTrackingData.add(new ExportMeasurement(tool.getName(), point.getX(), point.getY(), point.getZ(), shifted_points[0], shifted_points[1], shifted_points[2], x_normalized, y_normalized)); + data.add(new XYChart.Data<>(shifted_points[0], shifted_points[1])); + lineData.add(new XYChart.Data<>(shifted_points[0], shifted_points[1])); + lineData.add(new XYChart.Data<>(referencePoint.getData().get(0).getXValue(), referencePoint.getData().get(0).getYValue())); + + lineSeries.setData(lineData); + if(data.size() > max_num_points){ data.remove(0); + //lineData.remove(0); } } } @@ -322,7 +397,7 @@ private void saveCapturedData(BufferedImage image, List track gson.toJson(trackingData, fw); } } catch (IOException e) { - logger.log(java.util.logging.Level.SEVERE, "Error saving captured data", e); + logger.log(Level.SEVERE, "Error saving captured data", e); e.printStackTrace(); } } @@ -354,7 +429,7 @@ public void on_openOutputDirectory() { try { Desktop.getDesktop().open(new File(directory)); } catch (IOException e) { - logger.log(java.util.logging.Level.SEVERE, "Error opening output directory", e); + logger.log(Level.SEVERE, "Error opening output directory", e); e.printStackTrace(); } } @@ -415,7 +490,7 @@ public void on_importMatrix() { cachedTransformMatrix = null; userPreferences.put("matrixDirectory", inputFile.getAbsoluteFile().getParent()); }catch (FileNotFoundException e) { - logger.log(java.util.logging.Level.SEVERE, "Error loading matrix", e); + logger.log(Level.SEVERE, "Error loading matrix", e); e.printStackTrace(); } } @@ -432,7 +507,7 @@ public void on_reloadMatrix(){ regMatrixStatusBox.setSelected(true); cachedTransformMatrix = null; }catch (FileNotFoundException e) { - logger.log(java.util.logging.Level.SEVERE, "Error loading matrix", e); + logger.log(Level.SEVERE, "Error loading matrix", e); e.printStackTrace(); } } @@ -462,7 +537,7 @@ public void on_generateMatrix(){ cachedTransformMatrix = null; userPreferences.put("matrixDirectory", saveFile.getAbsoluteFile().getParent()); } catch (IOException e) { - logger.log(java.util.logging.Level.SEVERE, "Error saving matrix", e); + logger.log(Level.SEVERE, "Error saving matrix", e); e.printStackTrace(); } } @@ -472,7 +547,7 @@ private File getLastKnownFileLocation(String key, String defaultLocation){ var lastLocation = userPreferences.get(key,defaultLocation); var lastLocationFile = new File(lastLocation); if(!lastLocationFile.exists()){ - logger.log(java.util.logging.Level.WARNING, "Last directory does not exist, default directory instead"); + logger.log(Level.WARNING, "Last directory does not exist, default directory instead"); lastLocationFile = new File(defaultLocation); } return lastLocationFile; @@ -580,6 +655,7 @@ private double[] applyTrackingTransformation3d(double x, double y, double z){ * @param y Y-Coordinate in the image */ private void onImageClicked(double x, double y){ + System.out.println("clicked on image"); var trackingData = lastTrackingData; if(trackingData.size() > 0 && clicked_image_points.size() < 4) { // We also directly save the tracking-coordinates at this point. diff --git a/src/main/java/controller/MainController.java b/src/main/java/controller/MainController.java index f2445842..74e99901 100644 --- a/src/main/java/controller/MainController.java +++ b/src/main/java/controller/MainController.java @@ -2,14 +2,20 @@ import java.io.IOException; import java.net.URL; +import java.util.Objects; import java.util.ResourceBundle; import java.util.logging.Level; import java.util.logging.Logger; import algorithm.VisualizationManager; import javafx.application.Platform; +import javafx.event.ActionEvent; import javafx.fxml.FXML; import javafx.fxml.FXMLLoader; +import javafx.scene.Node; import javafx.scene.Scene; +import javafx.scene.control.*; +import javafx.stage.Modality; +import javafx.stage.Stage; import javafx.scene.control.Alert; import javafx.scene.control.Label; import javafx.scene.control.Tab; @@ -17,6 +23,12 @@ import javafx.stage.Modality; import javafx.stage.Stage; +import java.io.IOException; +import java.net.URL; +import java.util.ResourceBundle; +import java.util.logging.Level; +import java.util.logging.Logger; + public class MainController implements Controller { @FXML @@ -38,6 +50,8 @@ public class MainController implements Controller { private MeasurementController measurementController; private ThrombectomyController thrombectomyController; private AutoTrackController autoTrackController; + + private AiController AiController; private SettingsController settingsController; private final VisualizationManager visualizationManager = new VisualizationManager(); private final Logger logger = Logger.getLogger(this.getClass().getName()); @@ -113,6 +127,28 @@ private void openAutoTrackView(){ } } + @FXML + private void openAIView(){ + if (this.AiController != null) return; + + try { + setupFXMLLoader("AiView"); + Tab t = new Tab("AiView", this.loader.load()); + + this.AiController = this.loader.getController(); + this.AiController.setStatusLabel(this.status); + + this.tabPane.getTabs().add(t); + this.tabPane.getSelectionModel().select(t); + t.setOnCloseRequest(e -> { + this.AiController.close(); + this.AiController = null; + }); + } catch(IOException e) { + logger.log(Level.SEVERE, "Error loading AutoTrack View", e); + } + } + @FXML private void openSettings() { try { @@ -176,4 +212,47 @@ public void openAboutView() { a.setContentText("This application was and currently is developed by students of THU.\nIt is actively supervised by Prof. Dr. Alfred Franz.\nThe source code can be found at https://github.com/Alfred-Franz/IGTPrototypingTool"); a.showAndWait(); } + + + + /* + This function has the implementation of dark and light mode for the whole application + */ + @FXML + private void handleToggleTheme(ActionEvent event) { + try { + // Accessing the Scene from the MenuItem indirectly + MenuItem menuItem = (MenuItem) event.getSource(); + Scene scene = menuItem.getParentPopup().getOwnerWindow().getScene(); + + String lightModeUrl = Objects.requireNonNull(getClass().getResource("/css/customstyle.css")).toExternalForm(); + String darkModeUrl = Objects.requireNonNull(getClass().getResource("/css/dark-mode.css")).toExternalForm(); + + System.out.println("Light Mode URL: " + lightModeUrl); + System.out.println("Dark Mode URL: " + darkModeUrl); + + if (lightModeUrl == null || darkModeUrl == null) { + throw new Exception("Theme CSS file(s) not found."); + } + + if (scene.getStylesheets().contains(darkModeUrl)) { + scene.getStylesheets().remove(darkModeUrl); + scene.getStylesheets().add(lightModeUrl); + } else { + scene.getStylesheets().remove(lightModeUrl); + scene.getStylesheets().add(darkModeUrl); + } + } catch (Exception e) { + e.printStackTrace(); + showAlert("Error", "Failed to toggle theme: " + e.getMessage()); + } + } + + private void showAlert(String title, String content) { + Alert alert = new Alert(Alert.AlertType.INFORMATION); + alert.setTitle(title); + alert.setHeaderText(null); + alert.setContentText(content); + alert.showAndWait(); + } } diff --git a/src/main/java/controller/TrackingDataController.java b/src/main/java/controller/TrackingDataController.java index 1960d63c..d8f4fe98 100644 --- a/src/main/java/controller/TrackingDataController.java +++ b/src/main/java/controller/TrackingDataController.java @@ -12,6 +12,7 @@ import java.util.logging.Logger; import algorithm.*; +import inputOutput.AIDataSource; import inputOutput.CSVFileReader; import inputOutput.OIGTTrackingDataSource; import javafx.animation.Animation; @@ -122,6 +123,26 @@ public void loadCSVFile() { } } + @FXML + public void loadAIData() { + System.out.println("AI DATA LOADING"); + + AIDataSource newSource = new AIDataSource(); + + if (trackingService.getTrackingDataSource() != null) { + disconnectSource(); + } + + try { + trackingService.changeTrackingSource(newSource); + sourceConnected.setValue(true); + visualizationController.setSourceConnected(true); + } catch (Exception e) { + logger.log(Level.SEVERE, "Error loading AI DATA", e); + statusLabel.setText("Error loading AI DATA SOURCE"); + } + } + /** * Connect via OpenIGTLink. */ diff --git a/src/main/java/inputOutput/AIDataSource.java b/src/main/java/inputOutput/AIDataSource.java new file mode 100644 index 00000000..699dad2a --- /dev/null +++ b/src/main/java/inputOutput/AIDataSource.java @@ -0,0 +1,22 @@ +package inputOutput; + +import java.util.ArrayList; + +public class AIDataSource extends AbstractTrackingDataSource { + + public AIDataSource() { + tempToolList = new ArrayList<>(); + } + @Override + public ArrayList update() { + TempTool testTool1 = new TempTool(); + testTool1.setData(1, 1, 1, 1, 1, 1, 1, 1, 1, "testTool"); + tempToolList.add(testTool1); + return tempToolList; + } + + @Override + public void closeConnection() { + + } +} diff --git a/src/main/java/inputOutput/OIGTTrackingDataSource.java b/src/main/java/inputOutput/OIGTTrackingDataSource.java index 3f2148a5..8fb44d5d 100644 --- a/src/main/java/inputOutput/OIGTTrackingDataSource.java +++ b/src/main/java/inputOutput/OIGTTrackingDataSource.java @@ -81,6 +81,7 @@ public void setValues(String n, TransformNR t) { newTempTool.setData(timestamp, valid, coordinate_x, coordinate_y, coordinate_z, rotation_x, rotation_y, rotation_z, rotation_r, name); + this.tempToolList.add(newTempTool); } diff --git a/src/main/java/util/AnnotationData.java b/src/main/java/util/AnnotationData.java new file mode 100644 index 00000000..980107ec --- /dev/null +++ b/src/main/java/util/AnnotationData.java @@ -0,0 +1,102 @@ +package util; + +import javafx.scene.paint.Color; +import javafx.scene.shape.Rectangle; + +import java.util.HashMap; +import java.util.Map; + +/** + * Annotation Data handler responsible for collecting and saving the annotated data + * Singleton because we only want one instance of this, at a time. + * + */ +public class AnnotationData { + + public interface PublicAnnotation { + double getMiddlePointX(); + double getMiddlePointY(); + double getBoundingBoxWidth(); + double getBoundingBoxHeight(); + } + /** + * Private Class to save the Annotation Data + */ + private static class Annotation implements PublicAnnotation{ + private final double middlePointX; + private final double middlePointY; + private final double boundingBoxWidth; + private final double boundingBoxHeight; + + public Annotation(double middlePointX, double middlePointY, double boundingBoxWidth, double boundingBoxHeight) { + this.middlePointX = middlePointX; + this.middlePointY = middlePointY; + this.boundingBoxWidth = boundingBoxWidth; + this.boundingBoxHeight = boundingBoxHeight; + } + public double getMiddlePointX() {return middlePointX;} + public double getMiddlePointY() {return middlePointY;} + public double getBoundingBoxWidth() {return boundingBoxWidth;} + public double getBoundingBoxHeight() {return boundingBoxHeight;} + } + + private static final AnnotationData instance = new AnnotationData(); + + private final Map annotations; + private AnnotationData(){ + annotations = new HashMap<>(); + } + + public void addAnnotation(String path, + double middlePointX, + double middlePointY, + double boundingBoxWidth, + double boundingBoxHeight){ + if(!annotations.containsKey(path)){ + annotations.put(path, new Annotation(middlePointX, middlePointY, boundingBoxWidth, boundingBoxHeight)); + }else{ + annotations.replace(path, new Annotation(middlePointX, middlePointY, boundingBoxWidth, boundingBoxHeight)); + } + } + + public Rectangle getAnnotation(String path){ + if(annotations.containsKey(path)){ + Annotation annotation = annotations.get(path); + // Calculate the Rectangle based on the annotated data + Rectangle rectangle = new Rectangle(); + rectangle.setWidth(annotation.getBoundingBoxWidth()); + rectangle.setHeight(annotation.getBoundingBoxHeight()); + rectangle.setX(annotation.getMiddlePointX()-(annotation.getBoundingBoxWidth()/2)); + rectangle.setY(annotation.getMiddlePointY()-(annotation.getBoundingBoxHeight()/2)); + rectangle.setFill(Color.TRANSPARENT); + rectangle.setStroke(Color.rgb(6, 207, 236)); + rectangle.setStrokeWidth(2); + rectangle.setVisible(true); + return rectangle; + } + return null; + } + + /** + * Function to delete the Annotation of one Image + * + * @param path Image path (Key for the Map) + * @return + */ + public boolean deleteAnnotation(String path){ + annotations.remove(path); + return false; + } + + public static AnnotationData getInstance() { + return instance; + } + + public Map getAnnotations() { + return new HashMap<>(annotations); + } + + public PublicAnnotation getAnnotationEntry(String path){ + return annotations.get(path); + } +} diff --git a/src/main/resources/css/dark-mode.css b/src/main/resources/css/dark-mode.css new file mode 100644 index 00000000..93bad55f --- /dev/null +++ b/src/main/resources/css/dark-mode.css @@ -0,0 +1,48 @@ +.root { + -fx-background-color: #424242; + -fx-text-fill: white; +} + +.button { + -fx-background-color: #333333; + -fx-text-fill: white; +} + +.scroll-pane { + -fx-background: #424242; + -fx-border-color: #333333; +} + +.border-pane { + -fx-background-color: #424242; +} + +.vbox { + -fx-background-color: #424242; +} +.scroll-bar { + -fx-background-color: #333333; +} + +.scroll-bar .thumb { + -fx-background-color: #555555; + -fx-background-insets: 2; + -fx-background-radius: 5; +} + +.scroll-bar .track { + -fx-background-color: #444444; + -fx-background-insets: 2; + -fx-background-radius: 5; +} + +.scroll-bar .increment-button, +.scroll-bar .decrement-button { + -fx-background-color: #333333; + -fx-background-insets: 2; + -fx-background-radius: 5; +} + +.red-green-checkbox{ + -fx-border-color: red; +} \ No newline at end of file diff --git a/src/main/resources/view/AiView.fxml b/src/main/resources/view/AiView.fxml new file mode 100644 index 00000000..678f06e7 --- /dev/null +++ b/src/main/resources/view/AiView.fxml @@ -0,0 +1,90 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/main/resources/view/AnnotationView.fxml b/src/main/resources/view/AnnotationView.fxml new file mode 100644 index 00000000..aa0862aa --- /dev/null +++ b/src/main/resources/view/AnnotationView.fxml @@ -0,0 +1,111 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + +
+ + diff --git a/src/main/resources/view/AutoTrackView.fxml b/src/main/resources/view/AutoTrackView.fxml index f4b5ba69..36208846 100644 --- a/src/main/resources/view/AutoTrackView.fxml +++ b/src/main/resources/view/AutoTrackView.fxml @@ -5,22 +5,34 @@ + - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + @@ -82,6 +94,12 @@ + + + + + + + + +
@@ -38,12 +42,17 @@ - + + + + + +
diff --git a/src/main/resources/view/TrackingDataView.fxml b/src/main/resources/view/TrackingDataView.fxml index 9ee47ab4..100d2386 100644 --- a/src/main/resources/view/TrackingDataView.fxml +++ b/src/main/resources/view/TrackingDataView.fxml @@ -100,6 +100,8 @@